Merge pull request #11338 from dannon/fstrings

Convert/standardize to fstrings across `lib`
This commit is contained in:
Marius van den Beek
2021-05-14 13:08:37 +02:00
committed by GitHub
587 changed files with 3390 additions and 3418 deletions
+12 -12
View File
@@ -45,7 +45,7 @@ class AdminActions:
# If this is a default quota, create the DefaultQuotaAssociation
if params.default != 'no':
self.app.quota_agent.set_default_quota(params.default, quota)
message = "Default quota '%s' has been created." % quota.name
message = f"Default quota '{quota.name}' has been created."
else:
# Create the UserQuotaAssociations
in_users = [self.sa_session.query(self.app.model.User).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_users)]
@@ -113,7 +113,7 @@ class AdminActions:
quota.operation = params.operation
self.sa_session.add(quota)
self.sa_session.flush()
message = "Quota '{}' is now '{}'.".format(quota.name, quota.operation + quota.display_amount)
message = f"Quota '{quota.name}' is now '{quota.operation + quota.display_amount}'."
return message
def _set_quota_default(self, quota, params):
@@ -125,19 +125,19 @@ class AdminActions:
message = f"Quota '{quota.name}' is now the default for {params.default} users."
else:
if quota.default:
message = "Quota '{}' is no longer the default for {} users.".format(quota.name, quota.default[0].type)
message = f"Quota '{quota.name}' is no longer the default for {quota.default[0].type} users."
for dqa in quota.default:
self.sa_session.delete(dqa)
self.sa_session.flush()
else:
message = "Quota '%s' is not a default." % quota.name
message = f"Quota '{quota.name}' is not a default."
return message
def _unset_quota_default(self, quota, params=None):
if not quota.default:
raise ActionInputError("Quota '%s' is not a default." % quota.name)
raise ActionInputError(f"Quota '{quota.name}' is not a default.")
else:
message = "Quota '{}' is no longer the default for {} users.".format(quota.name, quota.default[0].type)
message = f"Quota '{quota.name}' is no longer the default for {quota.default[0].type} users."
for dqa in quota.default:
self.sa_session.delete(dqa)
self.sa_session.flush()
@@ -150,9 +150,9 @@ class AdminActions:
if q.default:
names.append(q.name)
if len(names) == 1:
raise ActionInputError("Quota '%s' is a default, please unset it as a default before deleting it." % (names[0]))
raise ActionInputError(f"Quota '{names[0]}' is a default, please unset it as a default before deleting it.")
elif len(names) > 1:
raise ActionInputError("Quotas are defaults, please unset them as defaults before deleting them: " + ', '.join(names))
raise ActionInputError(f"Quotas are defaults, please unset them as defaults before deleting them: {', '.join(names)}")
message = "Deleted %d quotas: " % len(quotas)
for q in quotas:
q.deleted = True
@@ -169,9 +169,9 @@ class AdminActions:
if not q.deleted:
names.append(q.name)
if len(names) == 1:
raise ActionInputError("Quota '%s' has not been deleted, so it cannot be undeleted." % (names[0]))
raise ActionInputError(f"Quota '{names[0]}' has not been deleted, so it cannot be undeleted.")
elif len(names) > 1:
raise ActionInputError("Quotas have not been deleted so they cannot be undeleted: " + ', '.join(names))
raise ActionInputError(f"Quotas have not been deleted so they cannot be undeleted: {', '.join(names)}")
message = "Undeleted %d quotas: " % len(quotas)
for q in quotas:
q.deleted = False
@@ -194,9 +194,9 @@ class AdminActions:
if not q.deleted:
names.append(q.name)
if len(names) == 1:
raise ActionInputError("Quota '%s' has not been deleted, so it cannot be purged." % (names[0]))
raise ActionInputError(f"Quota '{names[0]}' has not been deleted, so it cannot be purged.")
elif len(names) > 1:
raise ActionInputError("Quotas have not been deleted so they cannot be undeleted: " + ', '.join(names))
raise ActionInputError(f"Quotas have not been deleted so they cannot be undeleted: {', '.join(names)}")
message = "Purged %d quotas: " % len(quotas)
for q in quotas:
# Delete UserQuotaAssociations
+7 -7
View File
@@ -169,11 +169,11 @@ class LibraryActions:
if os.path.isfile(path):
files.append(path)
except Exception as e:
message = "Unable to get file list for configured {}, error: {}".format(import_dir_desc, util.unicodify(e))
message = f"Unable to get file list for configured {import_dir_desc}, error: {util.unicodify(e)}"
response_code = 500
return None, response_code, message
if not files:
message = "The directory '%s' contains no valid files" % full_dir
message = f"The directory '{full_dir}' contains no valid files"
response_code = 400
return None, response_code, message
return files, None, None
@@ -296,15 +296,15 @@ class LibraryActions:
if isinstance(item, trans.model.HistoryDatasetAssociation):
# Make sure the user has the DATASET_ACCESS permission on the history_dataset_association.
if not item:
message = "Invalid history dataset (%s) specified." % escape(str(item))
message = f"Invalid history dataset ({escape(str(item))}) specified."
can_access = False
elif not trans.app.security_agent.can_access_dataset(current_user_roles, item.dataset) and item.history.user == trans.user:
message = "You do not have permission to access the history dataset with id (%s)." % str(item.id)
message = f"You do not have permission to access the history dataset with id ({str(item.id)})."
can_access = False
else:
# Make sure the user has the LIBRARY_ACCESS permission on the library item.
if not item:
message = "Invalid library item (%s) specified." % escape(str(item))
message = f"Invalid library item ({escape(str(item))}) specified."
can_access = False
elif not (is_admin or trans.app.security_agent.can_access_library_item(current_user_roles, item, trans.user)):
if isinstance(item, trans.model.Library):
@@ -313,7 +313,7 @@ class LibraryActions:
item_type = 'folder'
else:
item_type = '(unknown item type)'
message = "You do not have permission to access the {} with id ({}).".format(escape(item_type), str(item.id))
message = f"You do not have permission to access the {escape(item_type)} with id ({str(item.id)})."
can_access = False
if not can_access:
return 400, message
@@ -321,5 +321,5 @@ class LibraryActions:
def _check_add(self, trans, is_admin, item, current_user_roles):
# Deny access if the user is not an admin and does not have the LIBRARY_ADD permission.
if not (is_admin or trans.app.security_agent.can_add_library_item(current_user_roles, item)):
message = "You are not authorized to add an item to (%s)." % escape(item.name)
message = f"You are not authorized to add an item to ({escape(item.name)})."
return 403, message
+1 -1
View File
@@ -350,7 +350,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self.url_for = url_for
self.server_starttime = int(time.time()) # used for cachebusting
log.info("Galaxy app startup finished %s" % startup_timer)
log.info(f"Galaxy app startup finished {startup_timer}")
def _shutdown_queue_worker(self):
self.queue_worker.shutdown()
+3 -3
View File
@@ -56,7 +56,7 @@ class AuthManager:
}
for provider, options in self.active_authenticators(email, username, password):
if provider is None:
log.debug("Unable to find module: %s" % options)
log.debug(f"Unable to find module: {options}")
else:
options['no_password_check'] = no_password_check
auth_results = provider.authenticate(email, username, password, options)
@@ -76,7 +76,7 @@ class AuthManager:
"""Checks the username/email and password using auth providers."""
for provider, options in self.active_authenticators(user.email, user.username, password):
if provider is None:
log.debug("Unable to find module: %s" % options)
log.debug(f"Unable to find module: {options}")
else:
auth_result = provider.authenticate_user(user, password, options)
if auth_result is True:
@@ -91,7 +91,7 @@ class AuthManager:
"""
for provider, options in self.active_authenticators(user.email, user.username, current_password):
if provider is None:
log.debug("Unable to find module: %s" % options)
log.debug(f"Unable to find module: {options}")
else:
auth_result = provider.authenticate_user(user, current_password, options)
if auth_result is True:
+1 -1
View File
@@ -26,7 +26,7 @@ class AlwaysReject(AuthProvider):
"""
See abstract method documentation.
"""
log.debug("User: %s, ALWAYSREJECT: None" % (user.id if options['redact_username_in_logs'] else user.email))
log.debug(f"User: {user.id if options['redact_username_in_logs'] else user.email}, ALWAYSREJECT: None")
return None
+1 -1
View File
@@ -25,7 +25,7 @@ log = logging.getLogger(__name__)
def _get_subs(d, k, params):
if k not in d or not d[k]:
raise ConfigurationError("Missing '%s' parameter in LDAP options" % k)
raise ConfigurationError(f"Missing '{k}' parameter in LDAP options")
return str(d[k]).format(**params)
+1 -1
View File
@@ -25,7 +25,7 @@ class LocalDB(AuthProvider):
See abstract method documentation.
"""
user_ok = user.check_password(password)
log.debug("User: {}, LOCALDB: {}".format(user.id if options['redact_username_in_logs'] else user.email, user_ok))
log.debug(f"User: {user.id if options['redact_username_in_logs'] else user.email}, LOCALDB: {user_ok}")
return user_ok
+4 -4
View File
@@ -64,7 +64,7 @@ class PAM(AuthProvider):
auto_register_email = None
force_fail = False
if not options['redact_username_in_logs']:
log.debug("use username: {} use email {} email {} username {}".format(options.get('login-use-username'), options.get('login-use-email', False), email, username))
log.debug(f"use username: {options.get('login-use-username')} use email {options.get('login-use-email', False)} email {email} username {username}")
# check email based login first because if email exists in Galaxy DB
# we will be given the "public name" as username
if string_as_bool(options.get('login-use-email', False)) and email is not None:
@@ -95,7 +95,7 @@ class PAM(AuthProvider):
elif options.get('maildomain', None) is not None:
# we can register a user with this username and mail domain
# if auto registration is enabled
auto_register_email = '{}@{}'.format(username, options['maildomain'])
auto_register_email = f"{username}@{options['maildomain']}"
auto_register_username = username
else:
log.debug('PAM authenticate: username login selected but no username provided')
@@ -142,10 +142,10 @@ class PAM(AuthProvider):
authenticated = p_auth.authenticate(pam_username, password, service=pam_service)
if authenticated:
log.debug('PAM authentication successful for {}'.format('redacted' if options['redact_username_in_logs'] else pam_username))
log.debug(f"PAM authentication successful for {'redacted' if options['redact_username_in_logs'] else pam_username}")
return True, auto_register_email, auto_register_username
else:
log.debug('PAM authentication failed for {}'.format('redacted' if options['redact_username_in_logs'] else pam_username))
log.debug(f"PAM authentication failed for {'redacted' if options['redact_username_in_logs'] else pam_username}")
return False, '', ''
def authenticate_user(self, user, password, options):
+1 -1
View File
@@ -47,7 +47,7 @@ def get_authenticators(auth_config_file, auth_config_file_set):
type_elem_text = auth_elem.find('type').text
plugin_class = __plugins_dict.get(type_elem_text)
if not plugin_class:
raise Exception("Authenticator type '{}' not recognized, should be one of {}".format(type_elem_text, ', '.join(__plugins_dict)))
raise Exception(f"Authenticator type '{type_elem_text}' not recognized, should be one of {', '.join(__plugins_dict)}")
plugin = plugin_class()
# check filterelem
+11 -11
View File
@@ -110,11 +110,11 @@ class CustosAuthnz(IdentityProvider):
and len(trans.app.auth_manager.authenticators) == 0):
user = existing_user
else:
message = "There already exists a user with email %s. To associate this external login, you must first be logged in as that existing account." % email
message = f"There already exists a user with email {email}. To associate this external login, you must first be logged in as that existing account."
log.exception(message)
raise exceptions.AuthenticationFailed(message)
else:
login_redirect_url = login_redirect_url + 'root/login?confirm=true&custos_token=' + json.dumps(token)
login_redirect_url = f"{login_redirect_url}root/login?confirm=true&custos_token={json.dumps(token)}"
return login_redirect_url, None
custos_authnz_token = CustosAuthnzToken(user=user,
@@ -180,7 +180,7 @@ class CustosAuthnz(IdentityProvider):
# Find CustosAuthnzToken record for this provider (should only be one)
provider_tokens = [token for token in user.custos_auth if token.provider == self.config["provider"]]
if len(provider_tokens) == 0:
raise Exception("User is not associated with provider {}".format(self.config["provider"]))
raise Exception(f"User is not associated with provider {self.config['provider']}")
if len(provider_tokens) > 1:
for idx, token in enumerate(provider_tokens):
id_token_decoded = self._decode_token_no_signature(token.id_token)
@@ -190,13 +190,13 @@ class CustosAuthnz(IdentityProvider):
trans.sa_session.flush()
return True, "", disconnect_redirect_url
except Exception as e:
return False, "Failed to disconnect provider {}: {}".format(provider, util.unicodify(e)), None
return False, f"Failed to disconnect provider {provider}: {util.unicodify(e)}", None
def logout(self, trans, post_logout_redirect_url=None):
try:
redirect_url = self.config['end_session_endpoint']
if post_logout_redirect_url is not None:
redirect_url += "?redirect_uri={}".format(quote(post_logout_redirect_url))
redirect_url += f"?redirect_uri={quote(post_logout_redirect_url)}"
return redirect_url
except Exception as e:
log.error("Failed to generate logout redirect_url", exc_info=e)
@@ -224,12 +224,12 @@ class CustosAuthnz(IdentityProvider):
else:
client_secret = self.config['client_secret']
token_endpoint = self.config['token_endpoint']
clientIdAndSec = self.config['client_id'] + ":" + self.config['client_secret'] # for custos
clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}" # for custos
return oauth2_session.fetch_token(
token_endpoint,
client_secret=client_secret,
authorization_response=trans.request.url,
headers={"Authorization": "Basic %s" % util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}, # for custos
headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, # for custos
verify=self._get_verify_param())
def _get_userinfo(self, oauth2_session):
@@ -263,9 +263,9 @@ class CustosAuthnz(IdentityProvider):
self.config['credential_url'] = '/'.join([self.config['url'].rstrip('/'), 'credentials'])
self._get_custos_credentials()
# Set custos endpoints
clientIdAndSec = self.config['client_id'] + ":" + self.config['client_secret']
clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}"
eps = requests.get(self.config['well_known_oidc_config_uri'],
headers={"Authorization": "Basic %s" % util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))},
headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"},
verify=False, params={'client_id': self.config['client_id']})
well_known_oidc_config = eps.json()
self._load_well_known_oidc_config(well_known_oidc_config)
@@ -276,9 +276,9 @@ class CustosAuthnz(IdentityProvider):
self._load_well_known_oidc_config(well_known_oidc_config)
def _get_custos_credentials(self):
clientIdAndSec = self.config['client_id'] + ":" + self.config['client_secret']
clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}"
creds = requests.get(self.config['credential_url'],
headers={"Authorization": "Basic %s" % util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))},
headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"},
verify=False, params={'client_id': self.config['client_id']})
credentials = creds.json()
self.config['iam_client_secret'] = credentials['iam_client_secret']
+2 -2
View File
@@ -165,7 +165,7 @@ class PSAAuthnz(IdentityProvider):
on_the_fly_config(trans.sa_session)
self.config[setting_name('LOGIN_REDIRECT_URL')] = login_redirect_url
strategy = Strategy(trans.request, trans.session, Storage, self.config)
strategy.session_set(BACKENDS_NAME[self.config['provider']] + '_state', state_token)
strategy.session_set(f"{BACKENDS_NAME[self.config['provider']]}_state", state_token)
backend = self._load_backend(strategy, self.config['redirect_uri'])
redirect_url = do_complete(
backend,
@@ -355,7 +355,7 @@ def verify(strategy=None, response=None, details=None, **kwargs):
result = requests.post(
f"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{endpoint}:getIamPolicy",
headers={
'Authorization': 'Bearer {}'.format(response.get("access_token")),
'Authorization': f"Bearer {response.get('access_token')}",
'Accept': 'application/json'})
res = json.loads(result.content)
if result.status_code == requests.codes.ok:
+1 -1
View File
@@ -33,7 +33,7 @@ def recalculate_user_disk_usage(session: scoped_session, user_id=None):
user.calculate_and_set_disk_usage()
log.info(f"New user disk usage is {user.disk_usage}")
else:
log.error("Recalculate user disk usage task failed, user %s not found" % user_id)
log.error(f"Recalculate user disk usage task failed, user {user_id} not found")
else:
log.error("Recalculate user disk usage task received without user_id.")
+15 -15
View File
@@ -176,7 +176,7 @@ class BaseAppConfiguration:
# NOTE: This will check all supplied keyword arguments, including those not in the schema.
# To check only schema options, change the line below to `if property not in self._raw_config:`
if key not in self._raw_config:
log.warning("Configuration option does not exist: '%s'" % key)
log.warning(f"Configuration option does not exist: '{key}'")
return key in self._kwargs
def resolve_path(self, path):
@@ -257,7 +257,7 @@ class BaseAppConfiguration:
for key in self.add_sample_file_to_defaults:
if not self.is_set(key):
defaults = listify(getattr(self, key), do_strip=True)
sample = '%s.sample' % defaults[-1] # if there are multiple defaults, use last as template
sample = f'{defaults[-1]}.sample' # if there are multiple defaults, use last as template
sample = self._in_sample_dir(sample) # resolve w.r.t sample_dir
defaults.append(sample)
setattr(self, key, defaults)
@@ -356,7 +356,7 @@ class BaseAppConfiguration:
if not hasattr(self, key):
setattr(self, key, value)
elif key not in base_configs:
raise ConfigurationError("Attempting to override existing attribute '%s'" % key)
raise ConfigurationError(f"Attempting to override existing attribute '{key}'")
def _resolve_paths(self):
@@ -486,7 +486,7 @@ class CommonConfigurationMixin:
try:
os.makedirs(path)
except Exception as e:
raise ConfigurationError("Unable to create missing directory: {}\n{}".format(path, unicodify(e)))
raise ConfigurationError(f"Unable to create missing directory: {path}\n{unicodify(e)}")
class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
@@ -581,7 +581,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.check_migrate_databases = kwargs.get('check_migrate_databases', True)
if not self.database_connection: # Provide default if not supplied by user
db_path = self._in_data_dir('universe.sqlite')
self.database_connection = 'sqlite:///%s?isolation_level=IMMEDIATE' % db_path
self.database_connection = f'sqlite:///{db_path}?isolation_level=IMMEDIATE'
self.database_engine_options = get_database_engine_options(kwargs)
self.database_create_tables = string_as_bool(kwargs.get('database_create_tables', 'True'))
self.database_encoding = kwargs.get('database_encoding') # Create new databases with this encoding
@@ -815,16 +815,16 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.amqp_internal_connection = kwargs.get('amqp_internal_connection')
# TODO Get extra amqp args as necessary for ssl
elif 'database_connection' in kwargs:
self.amqp_internal_connection = "sqlalchemy+" + self.database_connection
self.amqp_internal_connection = f"sqlalchemy+{self.database_connection}"
else:
self.amqp_internal_connection = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % self._in_data_dir("control.sqlite")
self.amqp_internal_connection = f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE"
self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format)
try:
with open(self.user_preferences_extra_conf_path) as stream:
self.user_preferences_extra = yaml.safe_load(stream)
except Exception:
if self.is_set('user_preferences_extra_conf_path'):
log.warning('Config file (%s) could not be found or is malformed.' % self.user_preferences_extra_conf_path)
log.warning(f'Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed.')
self.user_preferences_extra = {'preferences': {}}
# Experimental: This will not be enabled by default and will hide
@@ -840,7 +840,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
ie_dirs = self.interactive_environment_plugins_directory
self.gie_dirs = [d.strip() for d in (ie_dirs.split(",") if ie_dirs else [])]
if ie_dirs:
self.visualization_plugins_directory += ",%s" % ie_dirs
self.visualization_plugins_directory += f",{ie_dirs}"
self.proxy_session_map = self.dynamic_proxy_session_map
self.manage_dynamic_proxy = self.dynamic_proxy_manage # Set to false if being launched externally
@@ -991,14 +991,14 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
tool_configs = self.tool_configs
for path in tool_configs:
if not os.path.exists(path) and path not in (self.shed_tool_config_file, self.migrated_tools_config):
raise ConfigurationError("Tool config file not found: %s" % path)
raise ConfigurationError(f"Tool config file not found: {path}")
for datatypes_config in listify(self.datatypes_config):
if not os.path.isfile(datatypes_config):
raise ConfigurationError("Datatypes config file not found: %s" % datatypes_config)
raise ConfigurationError(f"Datatypes config file not found: {datatypes_config}")
# Check for deprecated options.
for key in self.config_dict.keys():
if key in self.deprecated_options:
log.warning("Config option '%s' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file." % key)
log.warning(f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file.")
@staticmethod
def _parse_allowed_origin_hostnames(allowed_origin_hostnames):
@@ -1033,7 +1033,7 @@ def reload_config_options(current_config):
if current_config._raw_config[option] != modified_config[option]:
current_config._raw_config[option] = modified_config[option]
setattr(current_config, option, modified_config[option])
log.info('Reloaded %s' % option)
log.info(f'Reloaded {option}')
def get_database_engine_options(kwargs, model_prefix=''):
@@ -1052,7 +1052,7 @@ def get_database_engine_options(kwargs, model_prefix=''):
'pool_threadlocal': string_as_bool,
'server_side_cursors': string_as_bool
}
prefix = "%sdatabase_engine_option_" % model_prefix
prefix = f"{model_prefix}database_engine_option_"
prefix_len = len(prefix)
rval = {}
for key, value in kwargs.items():
@@ -1306,7 +1306,7 @@ class ConfiguresGalaxyMixin:
else:
from galaxy.model.tool_shed_install import mapping as install_mapping
install_db_url = self.config.install_database_connection
log.info("Install database using its own connection %s" % install_db_url)
log.info(f"Install database using its own connection {install_db_url}")
self.install_model = install_mapping.init(install_db_url,
install_database_options)
+16 -16
View File
@@ -201,29 +201,29 @@ class _OptionAction:
class _DeprecatedAction(_OptionAction):
def lint(self, args, app_desc, key, value):
print("Option [%s] has been deprecated, this will likely be dropped in future releases of Galaxy." % key)
print(f"Option [{key}] has been deprecated, this will likely be dropped in future releases of Galaxy.")
class _DeprecatedAndDroppedAction(_OptionAction):
def converted(self, args, app_desc, key, value):
print("Option [%s] has been deprecated and dropped. It is not included in converted configuration." % key)
print(f"Option [{key}] has been deprecated and dropped. It is not included in converted configuration.")
return DROP_OPTION_VALUE
def lint(self, args, app_desc, key, value):
print("Option [%s] has been deprecated. Option should be dropped without replacement." % key)
print(f"Option [{key}] has been deprecated. Option should be dropped without replacement.")
class _PasteAppFactoryAction(_OptionAction):
def converted(self, args, app_desc, key, value):
if value not in app_desc.expected_app_factories:
raise Exception("Ending convert process - unknown paste factory encountered [%s]" % value)
raise Exception(f"Ending convert process - unknown paste factory encountered [{value}]")
return DROP_OPTION_VALUE
def lint(self, args, app_desc, key, value):
if value not in app_desc.expected_app_factories:
print("Problem - unknown paste app factory encountered [%s]" % value)
print(f"Problem - unknown paste app factory encountered [{value}]")
class _ProductionUnsafe(_OptionAction):
@@ -409,7 +409,7 @@ def _to_rst(args, app_desc, heading_level="~"):
def _write_option_rst(args, rst, key, heading_level, option_value):
title = "``%s``" % key
title = f"``{key}``"
heading = heading_level * len(title)
rst.write(f"{heading}\n{title}\n{heading}\n\n")
option, value = _parse_option_value(option_value)
@@ -426,9 +426,9 @@ def _write_option_rst(args, rst, key, heading_level, option_value):
default = "false"
elif default == "":
default = '""'
rst.write(":Default: ``%s``\n" % default)
rst.write(f":Default: ``{default}``\n")
if type:
rst.write(":Type: %s\n" % type)
rst.write(f":Type: {type}\n")
rst.write("\n\n")
@@ -513,7 +513,7 @@ def _find_app_options_from_config_parser(p):
def _lint(args, app_desc):
path = _find_config(args, app_desc)
if not os.path.exists(path):
raise Exception("Expected configuration file [%s] not found." % path)
raise Exception(f"Expected configuration file [{path}] not found.")
app_items = _find_app_options(app_desc, path)
for key, value in app_items.items():
option_action = OPTION_ACTIONS.get(key)
@@ -564,7 +564,7 @@ class GzipFilter:
def _run_conversion(args, app_desc):
ini_config = _find_config(args, app_desc)
if ini_config and not _is_ini(ini_config):
_warn("Cannot convert YAML file %s, this option is only for ini config files." % ini_config)
_warn(f"Cannot convert YAML file {ini_config}, this option is only for ini config files.")
sys.exit(1)
elif not ini_config:
_warn("Failed to find a config to convert - exiting without changes.")
@@ -613,7 +613,7 @@ def _run_conversion(args, app_desc):
if value in filters:
applied_filters.append(filters[value])
else:
_warn("Unknown filter found [%s], exiting..." % value)
_warn(f"Unknown filter found [{value}], exiting...")
sys.exit(1)
uwsgi_dict = _server_paste_to_uwsgi(app_desc, server_config, applied_filters)
@@ -656,7 +656,7 @@ def _is_ini(path):
def _replace_file(args, f, app_desc, from_path, to_path):
_write_to_file(args, f, to_path)
backup_path = "%s.backup" % from_path
backup_path = f"{from_path}.backup"
print(f"Moving [{from_path}] to [{backup_path}]")
if args.dry_run:
print("... skipping because --dry-run is enabled.")
@@ -704,7 +704,7 @@ def _write_to_file(args, f, path):
print(f"Overwriting {path} with the following contents:\n{contents_indented}")
print("... skipping because --dry-run is enabled.")
else:
print("Overwriting %s" % path)
print(f"Overwriting {path}")
safe_makedirs(os.path.dirname(path))
with open(path, "w") as to_f:
to_f.write(contents)
@@ -736,7 +736,7 @@ def _write_section(args, f, section_header, section_dict, uwsgi_hack=False):
def _write_header(f, section_header):
f.write("%s:\n\n" % section_header)
f.write(f"{section_header}:\n\n")
def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False):
@@ -756,7 +756,7 @@ def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False
key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}")
lines = f"{comment}{as_comment_str}{key_val_str}"
lines_idented = "\n".join(f" {line}" for line in lines.split("\n"))
f.write("%s\n\n" % lines_idented)
f.write(f"{lines_idented}\n\n")
def _parse_option_value(option_value):
@@ -807,7 +807,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
def _warn(message):
print("WARNING: %s" % message)
print(f"WARNING: {message}")
def _get_option_desc(option):
+4 -4
View File
@@ -90,7 +90,7 @@ def main(argv=None):
if not os.path.exists(directory):
os.makedirs(directory)
print("Bootstrapping Galaxy configuration into directory %s" % relative_config_dir)
print(f"Bootstrapping Galaxy configuration into directory {relative_config_dir}")
_handle_galaxy_yml(args, config_dir, data_dir)
_handle_install(args, dependencies)
_print_config_summary(args, mode, relative_config_dir)
@@ -104,7 +104,7 @@ def _print_config_summary(args, mode, relative_config_dir):
def _print_galaxy_yml_info(args, mode):
print(" - galaxy.yml created, update to configure Galaxy.")
print(" * Target web server %s" % mode)
print(f" * Target web server {mode}")
if args.host == DEFAULT_HOST:
print(" * Binding to host localhost, remote clients will not be able to connect.")
elif _determine_host(args) == "0.0.0.0":
@@ -117,7 +117,7 @@ def _print_galaxy_run(mode):
if mode.startswith("uwsgi"):
print(" uwsgi --yaml galaxy.yml")
else:
raise Exception("Unknown mode: %s" % mode)
raise Exception(f"Unknown mode: {mode}")
def _determine_mode(args):
@@ -178,7 +178,7 @@ def _handle_install(args, dependencies):
def _check_file(path, force):
if os.path.exists(path) and not force:
print("File %s exists, exiting. Run with --force to replace configuration." % path, file=sys.stderr)
print(f"File {path} exists, exiting. Run with --force to replace configuration.", file=sys.stderr)
sys.exit(1)
+8 -8
View File
@@ -45,7 +45,7 @@ class ContainerVolume(metaclass=ABCMeta):
self.host_path = host_path
self.mode = mode
if mode and not self.mode_is_valid:
raise ValueError("Invalid container volume mode: %s" % mode)
raise ValueError(f"Invalid container volume mode: {mode}")
@abstractmethod
def from_str(cls, as_str):
@@ -205,7 +205,7 @@ class ContainerInterface(metaclass=ABCMeta):
return opttype
def _guess_kwopt_flag(self, opt):
return '--%s' % opt.replace('_', '-')
return f"--{opt.replace('_', '-')}"
def _stringify_kwopts(self, kwopts):
opts = []
@@ -219,25 +219,25 @@ class ContainerInterface(metaclass=ABCMeta):
}
log.warning("option '%s' not in %s.option_map, guessing flag '%s' type '%s'",
opt, self.__class__.__name__, optdef['flag'], optdef['type'])
opts.append(getattr(self, '_stringify_kwopt_' + optdef['type'])(optdef['flag'], val))
opts.append(getattr(self, f"_stringify_kwopt_{optdef['type']}")(optdef['flag'], val))
return ' '.join(opts)
def _stringify_kwopt_boolean(self, flag, val):
"""
"""
return '{flag}={value}'.format(flag=flag, value=str(val).lower())
return f'{flag}={str(val).lower()}'
def _stringify_kwopt_string(self, flag, val):
"""
"""
return '{flag} {value}'.format(flag=flag, value=shlex.quote(str(val)))
return f'{flag} {shlex.quote(str(val))}'
def _stringify_kwopt_list(self, flag, val):
"""
"""
if isinstance(val, str):
return self._stringify_kwopt_string(flag, val)
return ' '.join('{flag} {value}'.format(flag=flag, value=shlex.quote(str(v))) for v in val)
return ' '.join(f'{flag} {shlex.quote(str(v))}' for v in val)
def _stringify_kwopt_list_of_kvpairs(self, flag, val):
"""
@@ -272,7 +272,7 @@ class ContainerInterface(metaclass=ABCMeta):
return stdout.strip()
else:
msg = f"Command '{command}' returned non-zero exit status {p.returncode}"
log.error(msg + ': ' + stderr.strip())
log.error(f"{msg}: {stderr.strip()}")
raise ContainerCLIError(
msg,
stdout=stdout.strip(),
@@ -346,7 +346,7 @@ def build_container_interfaces(containers_config_file, containers_conf=None):
interfaces = {}
for k, conf in containers_conf.items():
container_type = conf.get('type', DEFAULT_CONTAINER_TYPE)
assert container_type in interface_classes, "unknown container interface type: %s" % container_type
assert container_type in interface_classes, f"unknown container interface type: {container_type}"
interfaces[k] = interface_classes[container_type](conf, k, containers_config_file)
return interfaces
+7 -7
View File
@@ -168,7 +168,7 @@ class DockerCLIInterface(DockerInterface):
kwopt_list.append('{vol}:{bind}{mode}'.format(
vol=hostvol,
bind=guestopts['bind'],
mode=':' + mode if mode else ''
mode=f":{mode}" if mode else ''
))
return self._stringify_kwopt_list(flag, kwopt_list)
@@ -198,7 +198,7 @@ class DockerCLIInterface(DockerInterface):
try:
return self._run_docker(subcommand='inspect', args=container_id)[0]
except (IndexError, ContainerCLIError) as exc:
msg = "Invalid container id: %s" % container_id
msg = f"Invalid container id: {container_id}"
if exc.stdout == '[]' and exc.stderr == f'Error: no such object: {container_id}':
log.warning(msg)
return []
@@ -210,7 +210,7 @@ class DockerCLIInterface(DockerInterface):
try:
return self._run_docker(subcommand='image inspect', args=image)[0]
except (IndexError, ContainerCLIError) as exc:
msg = "%s not pulled, cannot get digest" % image
msg = f"{image} not pulled, cannot get digest"
if exc.stdout == '[]' and exc.stderr == f'Error: no such image: {image}':
log.warning(msg, image)
return []
@@ -234,7 +234,7 @@ class DockerAPIClient:
if isinstance(f, partial):
f = f.func
try:
return getattr(f, '__qualname__', f.im_class.__name__ + '.' + f.__name__)
return getattr(f, '__qualname__', f"{f.im_class.__name__}.{f.__name__}")
except AttributeError:
return f.__name__
@@ -489,7 +489,7 @@ class DockerAPIInterface(DockerInterface):
# keyword arguments
spec_kwopts = {}
# retrieve the option map for the docker-py object we're creating
option_map = getattr(self, option_map_name + '_option_map')
option_map = getattr(self, f"{option_map_name}_option_map")
# set defaults
for key in filter(lambda k: option_map[k].get('default'), option_map.keys()):
map_spec = option_map[key]
@@ -585,10 +585,10 @@ class DockerAPIInterface(DockerInterface):
try:
return self._client.inspect_container(container_id)
except docker.errors.NotFound:
raise ContainerNotFound("Invalid container id: %s" % container_id, container_id=container_id)
raise ContainerNotFound(f"Invalid container id: {container_id}", container_id=container_id)
def image_inspect(self, image):
try:
return self._client.inspect_image(image)
except docker.errors.NotFound:
raise ContainerImageNotFound("%s not pulled, cannot get digest" % image, image=image)
raise ContainerImageNotFound(f"{image} not pulled, cannot get digest", image=image)
+6 -6
View File
@@ -24,8 +24,8 @@ from galaxy.util import (
CPUS_LABEL = '_galaxy_cpus'
IMAGE_LABEL = '_galaxy_image'
CPUS_CONSTRAINT = 'node.labels.' + CPUS_LABEL
IMAGE_CONSTRAINT = 'node.labels.' + IMAGE_LABEL
CPUS_CONSTRAINT = f"node.labels.{CPUS_LABEL}"
IMAGE_CONSTRAINT = f"node.labels.{IMAGE_LABEL}"
log = logging.getLogger(__name__)
@@ -85,7 +85,7 @@ class DockerVolume(ContainerVolume):
Docker volume syntax.
"""
if not as_str:
raise ValueError("Failed to parse Docker volume from %s" % as_str)
raise ValueError(f"Failed to parse Docker volume from {as_str}")
parts = as_str.split(":", 2)
kwds = dict(host_path=parts[0])
if len(parts) == 1:
@@ -109,7 +109,7 @@ class DockerVolume(ContainerVolume):
volume_for_cmd_line = shlex.quote(volume_str)
else:
# e.g. $_GALAXY_JOB_TMP_DIR:$_GALAXY_JOB_TMP_DIR:rw so don't single quote.
volume_for_cmd_line = '"%s"' % volume_str
volume_for_cmd_line = f'"{volume_str}"'
return volume_for_cmd_line
def to_native(self):
@@ -406,7 +406,7 @@ class DockerServiceConstraint:
if len(t[0]) < len(constraint[0]):
constraint = t
if constraint[0] == constraint_str:
raise Exception('Unable to parse constraint string: %s' % constraint_str)
raise Exception(f'Unable to parse constraint string: {constraint_str}')
return [x.strip() for x in constraint]
@classmethod
@@ -676,7 +676,7 @@ class DockerTask:
service = service or interface.service(id=t.get('ServiceID'))
node = node or interface.node(id=t.get('NodeID'))
if service:
name = service.name + '.' + str(t['Slot'])
name = f"{service.name}.{str(t['Slot'])}"
else:
name = t['ID']
image = t['Spec']['ContainerSpec']['Image'].split('@', 1)[0], # remove pin
+2 -2
View File
@@ -426,10 +426,10 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
# service constraints
kwopts['constraint'] = kwopts.get('constraint', [])
if self._conf.service_create_image_constraint:
kwopts['constraint'].append(IMAGE_CONSTRAINT + '==' + image)
kwopts['constraint'].append(f"{IMAGE_CONSTRAINT}=={image}")
if self._conf.service_create_cpus_constraint:
cpus = kwopts.get('reserve_cpus', kwopts.get('limit_cpus', '1'))
kwopts['constraint'].append(CPUS_CONSTRAINT + '==' + cpus)
kwopts['constraint'].append(f"{CPUS_CONSTRAINT}=={cpus}")
# ports
if 'publish_port_random' in kwopts:
kwopts['ports'] = [DockerSwarmAPIInterface.create_random_port_spec(kwopts.pop('publish_port_random'))]
+2 -2
View File
@@ -26,7 +26,7 @@ class SnapHmm(Text):
try:
return dataset.peek
except Exception:
return "SNAP HMM model (%s)" % (nice_size(dataset.get_size()))
return f"SNAP HMM model ({nice_size(dataset.get_size())})"
def sniff_prefix(self, file_prefix):
"""
@@ -55,7 +55,7 @@ class Augustus(CompressedArchive):
try:
return dataset.peek
except Exception:
return "Augustus model (%s)" % (nice_size(dataset.get_size()))
return f"Augustus model ({nice_size(dataset.get_size())})"
def sniff(self, filename):
"""
+2 -2
View File
@@ -28,7 +28,7 @@ class AnvioComposite(Html):
cannot rename the datasets here - they come with the default unfortunately
"""
defined_files = self.get_composite_files(dataset=dataset).items()
rval = ["<html><head><title>Files for Anvi'o Composite Dataset (%s)</title></head>" % (self.file_ext)]
rval = [f"<html><head><title>Files for Anvi'o Composite Dataset ({self.file_ext})</title></head>"]
if defined_files:
rval.append("<p/>This composite dataset is composed of the following defined files:<p/><ul>")
for composite_name, composite_file in defined_files:
@@ -101,7 +101,7 @@ class AnvioDB(AnvioComposite):
if found:
break
if basename is not None and not os.path.exists(os.path.join(dataset.extra_files_path, basename)):
for name in glob.glob(os.path.join(dataset.extra_files_path, "*%s" % (basename))):
for name in glob.glob(os.path.join(dataset.extra_files_path, f"*{basename}")):
dataset.metadata.anvio_basename = os.path.basename(name)
found = True
break
+13 -13
View File
@@ -145,17 +145,17 @@ class Velvet(Html):
self.add_composite_file('Log', mimetype='text/html', description='Log', optional='True', substitute_name_with_metadata=None, is_binary=False)
def generate_primary_file(self, dataset=None):
log.debug("Velvet log info {} {}".format('JJ generate_primary_file', dataset))
log.debug(f"Velvet log info JJ generate_primary_file {dataset}")
rval = ['<html><head><title>Velvet Galaxy Composite Dataset </title></head><p/>']
rval.append('<div>This composite dataset is composed of the following files:<p/><ul>')
for composite_name, composite_file in self.get_composite_files(dataset=dataset).items():
fn = composite_name
log.debug("Velvet log info {} {} {}".format('JJ generate_primary_file', fn, composite_file))
log.debug(f"Velvet log info JJ generate_primary_file {fn} {composite_file}")
opt_text = ''
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"text/plain\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
@@ -165,7 +165,7 @@ class Velvet(Html):
"""
cannot do this until we are setting metadata
"""
log.debug("Velvet log info %s" % 'JJ regenerate_primary_file')
log.debug(f"Velvet log info {'JJ regenerate_primary_file'}")
gen_msg = ''
try:
efp = dataset.extra_files_path
@@ -173,7 +173,7 @@ class Velvet(Html):
with open(log_path) as f:
log_content = f.read(1000)
log_msg = re.sub(r'/\S*/', '', log_content)
log.debug("Velveth log info %s" % log_msg)
log.debug(f"Velveth log info {log_msg}")
paired_end_reads = re.search(r'-(short|long)Paired', log_msg) is not None
dataset.metadata.paired_end_reads = paired_end_reads
long_reads = re.search(r'-long', log_msg) is not None
@@ -182,27 +182,27 @@ class Velvet(Html):
dataset.metadata.short2_reads = short2_reads
dataset.info = re.sub(r'.*velveth \S+', 'hash_length', re.sub(r'\n', ' ', log_msg))
if paired_end_reads:
gen_msg = gen_msg + ' Paired-End Reads'
gen_msg = f"{gen_msg} Paired-End Reads"
if long_reads:
gen_msg = gen_msg + ' Long Reads'
gen_msg = f"{gen_msg} Long Reads"
if len(gen_msg) > 0:
gen_msg = 'Uses: ' + gen_msg
gen_msg = f"Uses: {gen_msg}"
except Exception:
log.debug("Velveth could not read Log file in %s" % efp)
log.debug("Velveth log info %s" % gen_msg)
log.debug(f"Velveth could not read Log file in {efp}")
log.debug(f"Velveth log info {gen_msg}")
rval = ['<html><head><title>Velvet Galaxy Composite Dataset </title></head><p/>']
# rval.append('<div>Generated:<p/><code> %s </code></div>' %(re.sub('\n','<br>',log_msg)))
rval.append('<div>Generated:<p/> %s </div>' % (gen_msg))
rval.append(f'<div>Generated:<p/> {gen_msg} </div>')
rval.append('<div>Velveth dataset:<p/><ul>')
for composite_name, composite_file in self.get_composite_files(dataset=dataset).items():
fn = composite_name
log.debug("Velvet log info {} {} {}".format('JJ regenerate_primary_file', fn, composite_file))
log.debug(f"Velvet log info JJ regenerate_primary_file {fn} {composite_file}")
if re.search('Log', fn) is None:
opt_text = ''
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"text/plain\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
+46 -46
View File
@@ -91,7 +91,7 @@ class Ab1(Binary):
try:
return dataset.peek
except Exception:
return "Binary ab1 sequence file (%s)" % (nice_size(dataset.get_size()))
return f"Binary ab1 sequence file ({nice_size(dataset.get_size())})"
class Idat(Binary):
@@ -166,7 +166,7 @@ class Cel(Binary):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.blurb = "Cel version: %s" % dataset.metadata.version
dataset.blurb = f"Cel version: {dataset.metadata.version}"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
@@ -205,7 +205,7 @@ class CompressedArchive(Binary):
try:
return dataset.peek
except Exception:
return "Compressed binary file (%s)" % (nice_size(dataset.get_size()))
return f"Compressed binary file ({nice_size(dataset.get_size())})"
class Meryldb(CompressedArchive):
@@ -283,7 +283,7 @@ class CompressedZipArchive(CompressedArchive):
try:
return dataset.peek
except Exception:
return "Compressed zip file (%s)" % (nice_size(dataset.get_size()))
return f"Compressed zip file ({nice_size(dataset.get_size())})"
def sniff(self, filename):
with zipfile.ZipFile(filename) as zf:
@@ -375,14 +375,14 @@ class BamNative(CompressedArchive):
try:
return dataset.peek
except Exception:
return "Binary bam alignments file (%s)" % (nice_size(dataset.get_size()))
return f"Binary bam alignments file ({nice_size(dataset.get_size())})"
def to_archive(self, dataset, name=""):
rel_paths = []
file_paths = []
rel_paths.append("{}.{}".format(name or dataset.file_name, dataset.extension))
rel_paths.append(f"{name or dataset.file_name}.{dataset.extension}")
file_paths.append(dataset.file_name)
rel_paths.append("{}.{}.bai".format(name or dataset.file_name, dataset.extension))
rel_paths.append(f"{name or dataset.file_name}.{dataset.extension}.bai")
file_paths.append(dataset.metadata.bam_index.file_name)
return zip(file_paths, rel_paths)
@@ -400,12 +400,12 @@ class BamNative(CompressedArchive):
return
tmp_dir = tempfile.mkdtemp()
tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix
sorted_file_name = f"{tmp_sorted_dataset_file_name_prefix}.bam"
slots = os.environ.get('GALAXY_SLOTS', 1)
sort_args = []
if self.sort_flag:
sort_args = [self.sort_flag]
sort_args.extend(["-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name])
sort_args.extend([f"-@{slots}", file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name])
try:
pysam.sort(*sort_args)
except Exception:
@@ -447,7 +447,7 @@ class BamNative(CompressedArchive):
offset = -1
except Exception as e:
offset = -1
ck_data = "Could not display BAM file, error was:\n%s" % e
ck_data = f"Could not display BAM file, error was:\n{e}"
else:
ck_data = ''
offset = -1
@@ -708,7 +708,7 @@ class CRAM(Binary):
def set_meta(self, dataset, overwrite=True, **kwd):
major_version, minor_version = self.get_cram_version(dataset.file_name)
if major_version != -1:
dataset.metadata.cram_version = str(major_version) + "." + str(minor_version)
dataset.metadata.cram_version = f"{str(major_version)}.{str(minor_version)}"
if not dataset.metadata.cram_index:
index_file = dataset.metadata.spec['cram_index'].param.new_file(dataset=dataset)
@@ -786,11 +786,11 @@ class Bcf(BaseBcf):
'__dataset_%d_%s' % (dataset.id, os.path.basename(index_file.file_name)))
os.symlink(dataset.file_name, dataset_symlink)
try:
cmd = ['python', '-c', "import pysam.bcftools; pysam.bcftools.index('%s')" % (dataset_symlink)]
cmd = ['python', '-c', f"import pysam.bcftools; pysam.bcftools.index('{dataset_symlink}')"]
subprocess.check_call(cmd)
shutil.move(dataset_symlink + '.csi', index_file.file_name)
shutil.move(f"{dataset_symlink}.csi", index_file.file_name)
except Exception as e:
raise Exception('Error setting BCF metadata: %s' % util.unicodify(e))
raise Exception(f'Error setting BCF metadata: {util.unicodify(e)}')
finally:
# Remove temp file and symlink
os.remove(dataset_symlink)
@@ -863,7 +863,7 @@ class H5(Binary):
try:
return dataset.peek
except Exception:
return "Binary HDF5 file (%s)" % (nice_size(dataset.get_size()))
return f"Binary HDF5 file ({nice_size(dataset.get_size())})"
class Loom(H5):
@@ -925,7 +925,7 @@ class Loom(H5):
try:
return dataset.peek
except Exception:
return "Binary Loom file (%s)" % (nice_size(dataset.get_size()))
return f"Binary Loom file ({nice_size(dataset.get_size())})"
def set_meta(self, dataset, overwrite=True, **kwd):
super().set_meta(dataset, overwrite=overwrite, **kwd)
@@ -1160,7 +1160,7 @@ class Anndata(H5):
peekstr += _makelayerstrings("uns", tmp.uns_count, tmp.uns_layers)
dataset.peek = peekstr
dataset.blurb = "Anndata file (%s)" % nice_size(dataset.get_size())
dataset.blurb = f"Anndata file ({nice_size(dataset.get_size())})"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -1169,7 +1169,7 @@ class Anndata(H5):
try:
return dataset.peek
except Exception:
return "Binary Anndata file (%s)" % (nice_size(dataset.get_size()))
return f"Binary Anndata file ({nice_size(dataset.get_size())})"
@build_sniff_from_prefix
@@ -1187,7 +1187,7 @@ class GmxBinary(Binary):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "Binary GROMACS %s file" % (self.file_ext)
dataset.peek = f"Binary GROMACS {self.file_ext} file"
dataset.blurb = nice_size(dataset.get_size())
else:
dataset.peek = 'file does not exist'
@@ -1197,7 +1197,7 @@ class GmxBinary(Binary):
try:
return dataset.peek
except Exception:
return "Binary GROMACS {} trajectory file ({})".format(self.file_ext, nice_size(dataset.get_size()))
return f"Binary GROMACS {self.file_ext} trajectory file ({nice_size(dataset.get_size())})"
class Trr(GmxBinary):
@@ -1330,7 +1330,7 @@ class Biom2(H5):
try:
with h5py.File(dataset.file_name) as f:
for k, v in f.attrs.items():
lines.append('{}: {}'.format(k, util.unicodify(v)))
lines.append(f'{k}: {util.unicodify(v)}')
except Exception as e:
log.warning('%s, set_peek Exception: %s', self, util.unicodify(e))
dataset.peek = '\n'.join(lines)
@@ -1343,7 +1343,7 @@ class Biom2(H5):
try:
return dataset.peek
except Exception:
return "Biom2 (HDF5) file (%s)" % (nice_size(dataset.get_size()))
return f"Biom2 (HDF5) file ({nice_size(dataset.get_size())})"
class Cool(H5):
@@ -1396,7 +1396,7 @@ class Cool(H5):
try:
return dataset.peek
except Exception:
return "Cool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
return f"Cool (HDF5) file ({nice_size(dataset.get_size())})."
class MCool(H5):
@@ -1456,7 +1456,7 @@ class MCool(H5):
try:
return dataset.peek
except Exception:
return "MCool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
return f"MCool (HDF5) file ({nice_size(dataset.get_size())})."
class H5MLM(H5):
@@ -1547,7 +1547,7 @@ class Scf(Binary):
try:
return dataset.peek
except Exception:
return "Binary scf sequence file (%s)" % (nice_size(dataset.get_size()))
return f"Binary scf sequence file ({nice_size(dataset.get_size())})"
@build_sniff_from_prefix
@@ -1574,7 +1574,7 @@ class Sff(Binary):
try:
return dataset.peek
except Exception:
return "Binary sff file (%s)" % (nice_size(dataset.get_size()))
return f"Binary sff file ({nice_size(dataset.get_size())})"
@build_sniff_from_prefix
@@ -1600,7 +1600,7 @@ class BigWig(Binary):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "Binary UCSC %s file" % self._name
dataset.peek = f"Binary UCSC {self._name} file"
dataset.blurb = nice_size(dataset.get_size())
else:
dataset.peek = 'file does not exist'
@@ -1610,7 +1610,7 @@ class BigWig(Binary):
try:
return dataset.peek
except Exception:
return "Binary UCSC {} file ({})".format(self._name, nice_size(dataset.get_size()))
return f"Binary UCSC {self._name} file ({nice_size(dataset.get_size())})"
class BigBed(BigWig):
@@ -1648,7 +1648,7 @@ class TwoBit(Binary):
try:
return dataset.peek
except Exception:
return "Binary TwoBit format nucleotide file (%s)" % (nice_size(dataset.get_size()))
return f"Binary TwoBit format nucleotide file ({nice_size(dataset.get_size())})"
@dataproviders.decorators.has_dataproviders
@@ -1675,7 +1675,7 @@ class SQlite(Binary):
for table, _ in rslt:
tables.append(table)
try:
col_query = 'SELECT * FROM %s LIMIT 0' % table
col_query = f'SELECT * FROM {table} LIMIT 0'
cur = conn.cursor().execute(col_query)
cols = [col[0] for col in cur.description]
columns[table] = cols
@@ -1683,7 +1683,7 @@ class SQlite(Binary):
log.warning('%s, set_meta Exception: %s', self, exc)
for table in tables:
try:
row_query = "SELECT count(*) FROM %s" % table
row_query = f"SELECT count(*) FROM {table}"
rowcounts[table] = c.execute(row_query).fetchone()[0]
except Exception as exc:
log.warning('%s, set_meta Exception: %s', self, exc)
@@ -1727,7 +1727,7 @@ class SQlite(Binary):
if dataset.metadata.tables:
for table in dataset.metadata.tables:
try:
lines.append('{} [{}]'.format(table, dataset.metadata.table_row_count[table]))
lines.append(f'{table} [{dataset.metadata.table_row_count[table]}]')
except Exception:
continue
dataset.peek = '\n'.join(lines)
@@ -1740,7 +1740,7 @@ class SQlite(Binary):
try:
return dataset.peek
except Exception:
return "SQLite Database (%s)" % (nice_size(dataset.get_size()))
return f"SQLite Database ({nice_size(dataset.get_size())})"
@dataproviders.decorators.dataprovider_factory('sqlite', dataproviders.dataset.SQliteDataProvider.settings)
def sqlite_dataprovider(self, dataset, **settings):
@@ -2027,7 +2027,7 @@ class DlibSQlite(SQlite):
c = conn.cursor()
tables_query = "SELECT Value FROM metadata WHERE Key = 'version'"
version = c.execute(tables_query).fetchall()[0]
dataset.metadata.dlib_version = '%s' % (version)
dataset.metadata.dlib_version = f'{version}'
except Exception as e:
log.warning('%s, set_meta Exception: %s', self, e)
@@ -2063,7 +2063,7 @@ class ElibSQlite(SQlite):
c = conn.cursor()
tables_query = "SELECT Value FROM metadata WHERE Key = 'version'"
version = c.execute(tables_query).fetchall()[0]
dataset.metadata.dlib_version = '%s' % (version)
dataset.metadata.dlib_version = f'{version}'
except Exception as e:
log.warning('%s, set_meta Exception: %s', self, e)
@@ -2119,7 +2119,7 @@ class IdpDB(SQlite):
try:
return dataset.peek
except Exception:
return "IDPickerDB SQLite file (%s)" % (nice_size(dataset.get_size()))
return f"IDPickerDB SQLite file ({nice_size(dataset.get_size())})"
class GAFASQLite(SQlite):
@@ -2246,7 +2246,7 @@ class ExcelXls(Binary):
try:
return dataset.peek
except Exception:
return "Microsoft Excel XLS file (%s)" % (data.nice_size(dataset.get_size()))
return f"Microsoft Excel XLS file ({data.nice_size(dataset.get_size())})"
@build_sniff_from_prefix
@@ -2272,7 +2272,7 @@ class Sra(Binary):
try:
return dataset.peek
except Exception:
return 'Binary sra file (%s)' % (nice_size(dataset.get_size()))
return f'Binary sra file ({nice_size(dataset.get_size())})'
class RData(Binary):
@@ -2494,7 +2494,7 @@ class PostgresqlArchive(CompressedArchive):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "PostgreSQL Archive (%s)" % (nice_size(dataset.get_size()))
dataset.peek = f"PostgreSQL Archive ({nice_size(dataset.get_size())})"
dataset.blurb = "PostgreSQL version %s" % (dataset.metadata.version or 'unknown')
else:
dataset.peek = 'file does not exist'
@@ -2504,7 +2504,7 @@ class PostgresqlArchive(CompressedArchive):
try:
return dataset.peek
except Exception:
return "PostgreSQL Archive (%s)" % (nice_size(dataset.get_size()))
return f"PostgreSQL Archive ({nice_size(dataset.get_size())})"
class Fast5Archive(CompressedArchive):
@@ -2548,7 +2548,7 @@ class Fast5Archive(CompressedArchive):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "FAST5 Archive (%s)" % (nice_size(dataset.get_size()))
dataset.peek = f"FAST5 Archive ({nice_size(dataset.get_size())})"
dataset.blurb = "%s sequences" % (dataset.metadata.fast5_count or 'unknown')
else:
dataset.peek = 'file does not exist'
@@ -2558,7 +2558,7 @@ class Fast5Archive(CompressedArchive):
try:
return dataset.peek
except Exception:
return "FAST5 Archive (%s)" % (nice_size(dataset.get_size()))
return f"FAST5 Archive ({nice_size(dataset.get_size())})"
class Fast5ArchiveGz(Fast5Archive):
@@ -2674,7 +2674,7 @@ class NetCDF(Binary):
try:
return dataset.peek
except Exception:
return "Binary netCDF file (%s)" % (nice_size(dataset.get_size()))
return f"Binary netCDF file ({nice_size(dataset.get_size())})"
def sniff_prefix(self, sniff_prefix):
return sniff_prefix.startswith_bytes(b'CDF')
@@ -2729,7 +2729,7 @@ class Dcd(Binary):
try:
return dataset.peek
except Exception:
return "Binary CHARMM/NAMD dcd file (%s)" % (nice_size(dataset.get_size()))
return f"Binary CHARMM/NAMD dcd file ({nice_size(dataset.get_size())})"
class Vel(Binary):
@@ -2780,7 +2780,7 @@ class Vel(Binary):
try:
return dataset.peek
except Exception:
return "Binary CHARMM velocity file (%s)" % (nice_size(dataset.get_size()))
return f"Binary CHARMM velocity file ({nice_size(dataset.get_size())})"
@build_sniff_from_prefix
@@ -2935,7 +2935,7 @@ class BafTar(CompressedArchive):
try:
return dataset.peek
except Exception:
return "{} ({})".format(self.get_type(), nice_size(dataset.get_size()))
return f"{self.get_type()} ({nice_size(dataset.get_size())})"
class YepTar(BafTar):
+10 -10
View File
@@ -103,40 +103,40 @@ class BlastXml(GenericXml):
old_header = None
for f in split_files:
if not os.path.isfile(f):
log.warning("BLAST XML file %s missing, retry in 1s..." % f)
log.warning(f"BLAST XML file {f} missing, retry in 1s...")
sleep(1)
if not os.path.isfile(f):
log.error("BLAST XML file %s missing" % f)
raise ValueError("BLAST XML file %s missing" % f)
log.error(f"BLAST XML file {f} missing")
raise ValueError(f"BLAST XML file {f} missing")
h = open(f)
header = h.readline()
if not header:
h.close()
# Retry, could be transient error with networked file system...
log.warning("BLAST XML file %s empty, retry in 1s..." % f)
log.warning(f"BLAST XML file {f} empty, retry in 1s...")
sleep(1)
h = open(f)
header = h.readline()
if not header:
log.error("BLAST XML file %s was empty" % f)
raise ValueError("BLAST XML file %s was empty" % f)
log.error(f"BLAST XML file {f} was empty")
raise ValueError(f"BLAST XML file {f} was empty")
if header.strip() != '<?xml version="1.0"?>':
out.write(header) # for diagnosis
h.close()
raise ValueError("%s is not an XML file!" % f)
raise ValueError(f"{f} is not an XML file!")
line = h.readline()
header += line
if line.strip() not in ['<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "http://www.ncbi.nlm.nih.gov/dtd/NCBI_BlastOutput.dtd">',
'<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "NCBI_BlastOutput.dtd">']:
out.write(header) # for diagnosis
h.close()
raise ValueError("%s is not a BLAST XML file!" % f)
raise ValueError(f"{f} is not a BLAST XML file!")
while True:
line = h.readline()
if not line:
out.write(header) # for diagnosis
h.close()
raise ValueError("BLAST XML file %s ended prematurely" % f)
raise ValueError(f"BLAST XML file {f} ended prematurely")
header += line
if "<Iteration>" in line:
break
@@ -145,7 +145,7 @@ class BlastXml(GenericXml):
# Write what we have to the merged file for diagnostics
out.write(header)
h.close()
raise ValueError("The header in BLAST XML file %s is too long" % f)
raise ValueError(f"The header in BLAST XML file {f} is too long")
if "<BlastOutput>" not in header:
h.close()
raise ValueError(f"{f} is not a BLAST XML file:\n{header}\n...")
@@ -107,7 +107,7 @@ class Ply:
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Faces: {}, Vertices: {}".format(str(dataset.metadata.face), str(dataset.metadata.vertex))
dataset.blurb = f"Faces: {str(dataset.metadata.face)}, Vertices: {str(dataset.metadata.vertex)}"
else:
dataset.peek = 'File does not exist'
dataset.blurb = 'File purged from disc'
@@ -116,7 +116,7 @@ class Ply:
try:
return dataset.peek
except Exception:
return "Ply file (%s)" % (nice_size(dataset.get_size()))
return f"Ply file ({nice_size(dataset.get_size())})"
class PlyAscii(Ply, data.Text):
@@ -420,7 +420,7 @@ class Vtk:
def get_blurb(self, dataset):
blurb = ""
if dataset.metadata.vtk_version is not None:
blurb += 'VTK Version %s' % str(dataset.metadata.vtk_version)
blurb += f'VTK Version {str(dataset.metadata.vtk_version)}'
if dataset.metadata.dataset_type is not None:
if blurb:
blurb += ' '
@@ -439,7 +439,7 @@ class Vtk:
try:
return dataset.peek
except Exception:
return "Vtk file (%s)" % (nice_size(dataset.get_size()))
return f"Vtk file ({nice_size(dataset.get_size())})"
class VtkAscii(Vtk, data.Text):
@@ -16,7 +16,7 @@ def main():
(options, args) = parser.parse_args()
input_fname, output_fname = args
slots = os.getenv('GALAXY_SLOTS', 1)
pysam.sort("-@%s" % slots, '-o', output_fname, '-O', 'bam', '-T', '.', input_fname)
pysam.sort(f"-@{slots}", '-o', output_fname, '-O', 'bam', '-T', '.', input_fname)
if __name__ == "__main__":
@@ -19,7 +19,7 @@ def __main__():
outfile = sys.argv[2]
if not os.path.isfile(infile):
sys.stderr.write("Input file %r not found\n" % infile)
sys.stderr.write(f"Input file {infile!r} not found\n")
sys.exit(1)
with open(infile) as inp:
@@ -39,7 +39,7 @@ def main():
line = in_file.readline()
chunk_end = in_file.tell()
out_file.write('{{"start":"{}","end":"{}","sequences":"{}"}}'.format(chunk_begin, chunk_end, (current_line % lines_per_chunk) / 4))
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{current_line % lines_per_chunk / 4}"}}')
out_file.write(']}\n')
@@ -19,7 +19,7 @@ assert sys.version_info[:2] >= (2, 4)
def stop_err(msg):
sys.exit("%s" % msg)
sys.exit(f"{msg}")
def __main__():
@@ -40,10 +40,10 @@ def __main__():
seq_title_startswith = line_startswith
if seq_title_startswith != line_startswith:
stop_err('Invalid fastqsolexa format at line %d: %s.' % (i + 1, line))
outfile.write('>%s\n' % line[1:])
outfile.write(f'>{line[1:]}\n')
elif fastq_block_lines == 2:
# line 2 is nucleotides
outfile.write('%s\n' % line)
outfile.write(f'{line}\n')
else:
pass
@@ -19,7 +19,7 @@ assert sys.version_info[:2] >= (2, 4)
def stop_err(msg):
sys.exit("%s" % msg)
sys.exit(f"{msg}")
def __main__():
@@ -57,9 +57,9 @@ def __main__():
if quality_title and read_title != quality_title:
stop_err('Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' % (i + 1, read_title, quality_title))
if not quality_title:
outfile_score.write('>%s\n' % read_title)
outfile_score.write(f'>{read_title}\n')
else:
outfile_score.write('>%s\n' % line[1:])
outfile_score.write(f'>{line[1:]}\n')
else:
# fourth line is quality scores
qual = ''
@@ -87,8 +87,8 @@ def __main__():
stop_err('Invalid fastqsolexa format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % (i + 1, quality_score_length, read_length))
for char in line:
score = ord(char) - quality_score_startswith # 64
qual = "{}{} ".format(qual, str(score))
outfile_score.write('%s\n' % qual)
qual = f"{qual}{str(score)} "
outfile_score.write(f'{qual}\n')
if __name__ == "__main__":
@@ -26,7 +26,7 @@ def __main__():
#
# Replace any spaces in the name with underscores so UCSC will not complain
name = elems[2].replace(" ", "_")
out.write("{}\t{}\t{}\t{}\t0\t{}\n".format(elems[0], start, elems[4], name, strand))
out.write(f"{elems[0]}\t{start}\t{elems[4]}\t{name}\t0\t{strand}\n")
except Exception:
skipped_lines += 1
if not first_skipped_line:
@@ -18,15 +18,15 @@ def __main__():
try:
chromCol = int(sys.argv[3]) - 1
except Exception:
stop_err("'%s' is an invalid chrom column, correct the column settings before attempting to convert the data format." % str(sys.argv[3]))
stop_err(f"'{str(sys.argv[3])}' is an invalid chrom column, correct the column settings before attempting to convert the data format.")
try:
startCol = int(sys.argv[4]) - 1
except Exception:
stop_err("'%s' is an invalid start column, correct the column settings before attempting to convert the data format." % str(sys.argv[4]))
stop_err(f"'{str(sys.argv[4])}' is an invalid start column, correct the column settings before attempting to convert the data format.")
try:
endCol = int(sys.argv[5]) - 1
except Exception:
stop_err("'%s' is an invalid end column, correct the column settings before attempting to convert the data format." % str(sys.argv[5]))
stop_err(f"'{str(sys.argv[5])}' is an invalid end column, correct the column settings before attempting to convert the data format.")
try:
strandCol = int(sys.argv[6]) - 1
except Exception:
@@ -40,15 +40,15 @@ def __main__():
try:
chromCol = int(sys.argv[3]) - 1
except Exception:
stop_err("'%s' is an invalid chrom column, correct the column settings before attempting to convert the data format." % str(sys.argv[3]))
stop_err(f"'{str(sys.argv[3])}' is an invalid chrom column, correct the column settings before attempting to convert the data format.")
try:
startCol = int(sys.argv[4]) - 1
except Exception:
stop_err("'%s' is an invalid start column, correct the column settings before attempting to convert the data format." % str(sys.argv[4]))
stop_err(f"'{str(sys.argv[4])}' is an invalid start column, correct the column settings before attempting to convert the data format.")
try:
endCol = int(sys.argv[5]) - 1
except Exception:
stop_err("'%s' is an invalid end column, correct the column settings before attempting to convert the data format." % str(sys.argv[5]))
stop_err(f"'{str(sys.argv[5])}' is an invalid end column, correct the column settings before attempting to convert the data format.")
try:
strandCol = int(sys.argv[6]) - 1
except Exception:
@@ -119,7 +119,7 @@ def __main__():
break
if force_num_columns is not None and len(fields) != force_num_columns:
line = '\t'.join(force_bed_field_count(fields, count, force_num_columns))
out.write("%s\n" % line)
out.write(f"{line}\n")
else:
strict_bed = False
@@ -97,9 +97,9 @@ def main():
# Write padded entries.
with open(out_fname, 'w') as out:
out.write(str(max_len + 1).ljust(max_len) + '\n')
out.write(f"{str(max_len + 1).ljust(max_len)}\n")
for entry in entries:
out.write(entry.ljust(max_len) + '\n')
out.write(f"{entry.ljust(max_len)}\n")
if __name__ == '__main__':
@@ -34,9 +34,9 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
"""convert linkage ped/map to fbat"""
recode = {'A': '1', 'C': '2', 'G': '3', 'T': '4', 'N': '0', '0': '0', '1': '1', '2': '2', '3': '3', '4': '4'}
basename = os.path.split(inpedfilepath)[-1] # get basename
inmap = '%s.map' % inpedfilepath
inped = '%s.ped' % inpedfilepath
outf = '%s.ped' % basename # note the fbat exe insists that this is the extension for the ped data
inmap = f'{inpedfilepath}.map'
inped = f'{inpedfilepath}.ped'
outf = f'{basename}.ped' # note the fbat exe insists that this is the extension for the ped data
outfpath = os.path.join(outfilepath, outf) # where to write the fbat format file to
try:
mf = open(inmap)
@@ -45,7 +45,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
try:
rsl = [x.split()[1] for x in mf]
except Exception:
sys.exit('## cannot parse %s' % inmap)
sys.exit(f'## cannot parse {inmap}')
try:
os.makedirs(outfilepath)
except Exception:
@@ -68,7 +68,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
g = lrow[6:]
gc = [recode.get(z, '0') for z in g]
lrow = p + gc
row = '%s\n' % ' '.join(lrow)
row = f"{' '.join(lrow)}\n"
o.write(row)
@@ -97,7 +97,7 @@ def main():
print(f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}') # becomes info
f.write(f'<div>## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}\n<ol>')
for data in flist:
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
f.write(f'<li><a href="{os.path.split(data)[-1]}">{os.path.split(data)[-1]}</a></li>\n')
f.write("</div></body></html>")
@@ -64,12 +64,12 @@ def getMissval(inped=''):
def rgConv(inpedfilepath, outhtmlname, outfilepath, plink):
"""
"""
pedf = '%s.ped' % inpedfilepath
pedf = f'{inpedfilepath}.ped'
basename = os.path.split(inpedfilepath)[-1] # get basename
outroot = os.path.join(outfilepath, basename)
missval = getMissval(inped=pedf)
if not missval:
print('### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf)
print(f'### lped_to_pbed_converter.py cannot identify missing value in {pedf}')
missval = '0'
subprocess.check_call([plink, '--noweb', '--file', inpedfilepath,
'--make-bed', '--out', outroot,
@@ -101,9 +101,9 @@ def main():
f.write(galhtmlprefix % prog)
s = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info
print(s)
f.write('<div>%s\n<ol>' % (s))
f.write(f'<div>{s}\n<ol>')
for data in flist:
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
f.write(f'<li><a href="{os.path.split(data)[-1]}">{os.path.split(data)[-1]}</a></li>\n')
f.write("</ol></div></div></body></html>")
@@ -24,7 +24,7 @@ def __main__():
else:
spec_counts[spec] += 1
out.write("%s\n" % maf_utilities.get_fasta_header(c, {'block_index': count, 'species': spec, 'sequence_index': spec_counts[spec]}, suffix="%s_%i_%i" % (spec, count, spec_counts[spec])))
out.write("%s\n" % c.text)
out.write(f"{c.text}\n")
out.write("\n")
print("%i MAF blocks converted to FASTA." % (count))
@@ -26,7 +26,7 @@ def __main__():
out.write("%s\t%i\t%i\t%s\n" % (maf_utilities.src_split(c.src)[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand))
count += 1
except Exception as e:
print("There was a problem processing your input: %s" % e, file=sys.stderr)
print(f"There was a problem processing your input: {e}", file=sys.stderr)
print("%i MAF blocks converted to Genomic Intervals for species %s." % (count, species))
@@ -47,7 +47,7 @@ def pruneLD(plinktasks=None, cd='./', vclbase=None):
alog += lplog
alog.append('\n')
except Exception:
alog.append('### {} Strange - no std out from plink when running command line\n{}\n'.format(timenow(), ' '.join(vcl)))
alog.append(f"### {timenow()} Strange - no std out from plink when running command line\n{' '.join(vcl)}\n")
return alog
@@ -59,7 +59,7 @@ def makeLDreduced(basename, infpath=None, outfpath=None, plinke='plink', forcere
inbase = os.path.join(infpath)
plinktasks = []
vclbase = [plinke, '--noweb']
plinktasks += [['--bfile', inbase, f'--indep-pairwise {winsize} {winmove} {r2thresh}', '--out %s' % outbase],
plinktasks += [['--bfile', inbase, f'--indep-pairwise {winsize} {winmove} {r2thresh}', f'--out {outbase}'],
['--bfile', inbase, f'--extract {outbase}.prune.in --make-bed --out {outbase}']]
vclbase = [plinke, '--noweb']
pruneLD(plinktasks=plinktasks, cd=outfpath, vclbase=vclbase)
@@ -103,7 +103,7 @@ def main():
print(f'{s1} {s2}')
f.write(f'<div>{s1}\n{s2}\n<ol>')
for data in flist:
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
f.write(f'<li><a href="{os.path.split(data)[-1]}">{os.path.split(data)[-1]}</a></li>\n')
f.write("</div></body></html>")
@@ -67,9 +67,9 @@ def main():
f.write(galhtmlprefix % prog)
s = f'## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {prog} {timenow()}' # becomes info
print(s)
f.write('<div>%s\n<ol>' % (s))
f.write(f'<div>{s}\n<ol>')
for data in flist:
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
f.write(f'<li><a href="{os.path.split(data)[-1]}">{os.path.split(data)[-1]}</a></li>\n')
f.write("</ol></div></div></body></html>")
@@ -23,7 +23,7 @@ def __main__():
else:
try:
elems = line.split('\t')
out.write('{}\t{}\t{}\t{}\t0\t{}\n'.format(elems[0], int(elems[1]) - 1, elems[2], elems[4], elems[3]))
out.write(f'{elems[0]}\t{int(elems[1]) - 1}\t{elems[2]}\t{elems[4]}\t0\t{elems[3]}\n')
except Exception as e:
print(e)
skipped_lines += 1
@@ -17,7 +17,7 @@ def __main__():
if line and not line.startswith('#'):
fields = line.split('\t')
# make sure the 2nd field (taxonomy) ends with a ;
outfile.write('{}\t{};\n'.format(fields[0], re.sub(';$', '', fields[1])))
outfile.write(f"{fields[0]}\t{re.sub(';$', '', fields[1])};\n")
if __name__ == "__main__":
+16 -16
View File
@@ -72,7 +72,7 @@ def validate(dataset_instance):
try:
datatype_validation = dataset_instance.datatype.validate(dataset_instance)
except Exception as e:
datatype_validation = DatatypeValidation.invalid("Problem running datatype validation method [%s]" % str(e))
datatype_validation = DatatypeValidation.invalid(f"Problem running datatype validation method [{str(e)}]")
return datatype_validation
@@ -244,11 +244,11 @@ class Data(metaclass=DataMeta):
line = line.strip()
if not line:
continue
out.append('<tr><td>%s</td></tr>' % escape(unicodify(line, 'utf-8')))
out.append(f"<tr><td>{escape(unicodify(line, 'utf-8'))}</td></tr>")
out.append('</table>')
out = "".join(out)
except Exception as exc:
out = "Can't create peek: %s" % unicodify(exc)
out = f"Can't create peek: {unicodify(exc)}"
return out
def _archive_main_file(self, archive, display_name, data_filename):
@@ -261,7 +261,7 @@ class Data(metaclass=DataMeta):
Returns a tuple of boolean, string, string: (error, msg, messagetype)
"""
error, msg, messagetype = False, "", ""
archname = '%s.html' % display_name # fake the real nature of the html file
archname = f'{display_name}.html' # fake the real nature of the html file
try:
archive.write(data_filename, archname)
except OSError:
@@ -319,7 +319,7 @@ class Data(metaclass=DataMeta):
trans.response.headers['Content-Length'] = str(os.stat(dataset.file_name).st_size)
trans.response.set_content_type("application/octet-stream") # force octet-stream so Safari doesn't append mime extensions to filename
filename = self._download_filename(dataset, to_ext, hdca=kwd.get("hdca"), element_identifier=kwd.get("element_identifier"), filename_pattern=kwd.get("filename_pattern"))
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s"' % filename
trans.response.headers["Content-Disposition"] = f'attachment; filename="{filename}"'
return open(dataset.file_name, mode='rb')
def to_archive(self, dataset, name=""):
@@ -386,7 +386,7 @@ class Data(metaclass=DataMeta):
# href = url_for(controller='dataset', action='display',
# dataset_id=trans.security.encode_id(data.dataset.id),
# preview=preview, filename=fname, to_ext=to_ext)
tmp_fh.write('<tr bgcolor="{}"><td>{}</td></tr>\n'.format(bgcolor, escape(fname)))
tmp_fh.write(f'<tr bgcolor="{bgcolor}"><td>{escape(fname)}</td></tr>\n')
tmp_fh.write('</table></body></html>\n')
return self._yield_user_file_content(trans, data, tmp_file_name)
mime = mimetypes.guess_type(file_path)[0]
@@ -401,7 +401,7 @@ class Data(metaclass=DataMeta):
return webob.exc.HTTPNotFound(f"Could not find '{filename}' on the extra files path {file_path}.")
self._clean_and_set_mime_type(trans, data.get_mime())
trans.log_event("Display dataset id: %s" % str(data.id))
trans.log_event(f"Display dataset id: {str(data.id)}")
from galaxy import datatypes # DBTODO REMOVE THIS AT REFACTOR
if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file
if data.extension in composite_extensions:
@@ -410,10 +410,10 @@ class Data(metaclass=DataMeta):
trans.response.headers['Content-Length'] = str(os.stat(data.file_name).st_size)
filename = self._download_filename(data, to_ext, hdca=kwd.get("hdca"), element_identifier=kwd.get("element_identifier"), filename_pattern=kwd.get("filename_pattern"))
trans.response.set_content_type("application/octet-stream") # force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s"' % filename
trans.response.headers["Content-Disposition"] = f'attachment; filename="{filename}"'
return open(data.file_name, 'rb')
if not os.path.exists(data.file_name):
raise webob.exc.HTTPNotFound("File Not Found (%s)." % data.file_name)
raise webob.exc.HTTPNotFound(f"File Not Found ({data.file_name}).")
max_peek_size = DEFAULT_MAX_PEEK_SIZE # 1 MB
if isinstance(data.datatype, datatypes.text.Html):
max_peek_size = 10000000 # 10 MB for html
@@ -639,7 +639,7 @@ class Data(metaclass=DataMeta):
# Run converter, job is dispatched through Queue
converted_dataset = converter.execute(trans, incoming=params, set_output_hid=visible, history=history)[1]
if len(params) > 0:
trans.log_event("Converter params: %s" % (str(params)), tool_id=converter.id)
trans.log_event(f"Converter params: {str(params)}", tool_id=converter.id)
if not visible:
for value in converted_dataset.values():
value.visible = False
@@ -727,7 +727,7 @@ class Data(metaclass=DataMeta):
max argument limitation of cat. gz and bz2 files are also working.
"""
if not split_files:
raise ValueError('Asked to merge zero files as %s' % output_file)
raise ValueError(f'Asked to merge zero files as {output_file}')
elif len(split_files) == 1:
shutil.copyfileobj(open(split_files[0], 'rb'), open(output_file, 'wb'))
else:
@@ -855,7 +855,7 @@ class Text(Data):
if line_count is None:
# See if line_count is stored in the metadata
if dataset.metadata.data_lines:
dataset.blurb = "{} {}".format(util.commaify(str(dataset.metadata.data_lines)), inflector.cond_plural(dataset.metadata.data_lines, self.line_class))
dataset.blurb = f"{util.commaify(str(dataset.metadata.data_lines))} {inflector.cond_plural(dataset.metadata.data_lines, self.line_class)}"
else:
# Number of lines is not known ( this should not happen ), and auto-detect is
# needed to set metadata
@@ -865,17 +865,17 @@ class Text(Data):
lc = self.count_data_lines(dataset)
if lc is not None:
dataset.metadata.data_lines = lc
dataset.blurb = "{} {}".format(util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class))
dataset.blurb = f"{util.commaify(str(lc))} {inflector.cond_plural(lc, self.line_class)}"
else:
dataset.blurb = "Error: Cannot count lines in dataset"
else:
est_lines = self.estimate_file_lines(dataset)
if est_lines is not None:
dataset.blurb = "~{} {}".format(util.commaify(util.roundify(str(est_lines))), inflector.cond_plural(est_lines, self.line_class))
dataset.blurb = f"~{util.commaify(util.roundify(str(est_lines)))} {inflector.cond_plural(est_lines, self.line_class)}"
else:
dataset.blurb = "Error: Cannot estimate lines in dataset"
else:
dataset.blurb = "{} {}".format(util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class))
dataset.blurb = f"{util.commaify(str(line_count))} {inflector.cond_plural(line_count, self.line_class)}"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -916,7 +916,7 @@ class Text(Data):
elif split_params['split_mode'] == 'to_size':
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
f = open(input_files[0])
try:
+1 -1
View File
@@ -160,7 +160,7 @@ class DataProvider(metaclass=HasSettings):
"""
# we need to protect against recursion (in __getattr__) if self.source hasn't been set
source_str = str(self.source) if hasattr(self, 'source') else ''
return '{}({})'.format(self.__class__.__name__, str(source_str))
return f'{self.__class__.__name__}({str(source_str)})'
class FilteredDataProvider(DataProvider):
@@ -126,7 +126,7 @@ class DatasetDataProvider(base.DataProvider):
or getattr(self.dataset.datatype, 'column_names', None) or None)
if not metadata_column_names:
raise KeyError('No column_names found for '
+ 'datatype: {}, dataset: {}'.format(str(self.dataset.datatype), str(self.dataset)))
+ f'datatype: {str(self.dataset.datatype)}, dataset: {str(self.dataset)}')
indeces = [] # if indeces and column_names:
# pull using indeces and re-name with given names - no need to alter (does as super would)
# pass
@@ -155,7 +155,7 @@ class DatasetDataProvider(base.DataProvider):
region_column_names = ('chromCol', 'startCol', 'endCol')
region_indices = [self.get_metadata_column_index_by_name(name) for name in region_column_names]
if check and not all(_ is not None for _ in region_indices):
raise ValueError("Could not determine proper column indices for chrom, start, end: %s" % (str(region_indices)))
raise ValueError(f"Could not determine proper column indices for chrom, start, end: {str(region_indices)}")
return region_indices
@@ -305,7 +305,7 @@ class GenomicRegionDataProvider(column.ColumnarDataProvider):
indeces = [chrom_column, start_column, end_column]
if not all(_ is not None for _ in indeces):
raise ValueError("Could not determine proper column indeces for"
+ " chrom, start, end: %s" % (str(indeces)))
+ f" chrom, start, end: {str(indeces)}")
kwargs.update({'indeces': indeces})
if not kwargs.get('column_types', None):
@@ -664,11 +664,11 @@ class SamtoolsDataProvider(line.RegexLineDataProvider):
validated_flag_list.append('S')
if validated_flag_list:
opt_list.append('-' + ''.join(validated_flag_list))
opt_list.append(f"-{''.join(validated_flag_list)}")
for flag, arg in options_dict.items():
if flag in self.FLAGS_W_ARGS:
opt_list.extend(['-' + flag, str(arg)])
opt_list.extend([f"-{flag}", str(arg)])
return opt_list
@@ -9,7 +9,7 @@ class InvalidDataProviderSource(TypeError):
"""
def __init__(self, source=None, msg=''):
msg = msg or 'Invalid source for provider: %s' % (source)
msg = msg or f'Invalid source for provider: {source}'
super().__init__(msg)
@@ -30,7 +30,7 @@ class NoProviderAvailable(TypeError):
def __init__(self, factory_source, format_requested=None, msg=''):
self.factory_source = factory_source
self.format_requested = format_requested
msg = msg or 'No provider available in factory_source "%s" for format requested' % (str(factory_source))
msg = msg or f'No provider available in factory_source "{str(factory_source)}" for format requested'
if self.format_requested:
msg += ': "%s"' % (self.format_requested)
msg += f': "{self.format_requested}"'
super().__init__(msg)
@@ -56,7 +56,7 @@ class SubprocessDataProvider(base.DataProvider):
try:
# how expensive is this?
popen = subprocess.Popen(command_list, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
log.info('opened subrocess ({}), PID: {}'.format(str(command_list), str(popen.pid)))
log.info(f'opened subrocess ({str(command_list)}), PID: {str(popen.pid)}')
except OSError as os_err:
command_str = ' '.join(self.command)
@@ -67,15 +67,15 @@ class SubprocessDataProvider(base.DataProvider):
def __exit__(self, *args):
# poll the subrocess for an exit code
self.exit_code = self.popen.poll()
log.info('{}.__exit__, exit_code: {}'.format(str(self), str(self.exit_code)))
log.info(f'{str(self)}.__exit__, exit_code: {str(self.exit_code)}')
return super().__exit__(*args)
def __str__(self):
# provide the pid and current return code
source_str = ''
if hasattr(self, 'popen'):
source_str = '{}:{}'.format(str(self.popen.pid), str(self.popen.poll()))
return '{}({})'.format(self.__class__.__name__, str(source_str))
source_str = f'{str(self.popen.pid)}:{str(self.popen.poll())}'
return f'{self.__class__.__name__}({str(source_str)})'
class RegexSubprocessDataProvider(line.RegexLineDataProvider):
@@ -115,15 +115,15 @@ class URLDataProvider(base.DataProvider):
encoded_data = urlencode(self.data)
scheme = urlparse(url).scheme
assert scheme in ('http', 'https', 'ftp'), 'Invalid URL scheme: %s' % scheme
assert scheme in ('http', 'https', 'ftp'), f'Invalid URL scheme: {scheme}'
if method == 'GET':
self.url += '?%s' % (encoded_data)
self.url += f'?{encoded_data}'
opened = urlopen(url)
elif method == 'POST':
opened = urlopen(url, encoded_data)
else:
raise ValueError('Not a valid method: %s' % (method))
raise ValueError(f'Not a valid method: {method}')
super().__init__(opened, **kwargs)
# NOTE: the request object is now accessible as self.source
@@ -172,4 +172,4 @@ class TempfileDataProvider(base.DataProvider):
parent_gen = super().__iter__()
with open(self.tmp_file, 'w') as open_file:
for datum in parent_gen:
open_file.write(datum + '\n')
open_file.write(f"{datum}\n")
@@ -38,7 +38,7 @@ class DisplayApplicationLink:
rval.filters = elem.findall('filter')
for param_elem in elem.findall('param'):
param = DisplayApplicationParameter.from_elem(param_elem, rval)
assert param, 'Unable to load parameter from element: %s' % param_elem
assert param, f'Unable to load parameter from element: {param_elem}'
rval.parameters[param.name] = param
rval.url_param_name_map[param.url] = param.name
return rval
@@ -79,7 +79,7 @@ class DisplayApplicationLink:
other_values['USER_HASH'] = user_hash
ready = True
for name, param in self.parameters.items():
assert name not in other_values, "The display parameter '%s' has been defined more than once." % name
assert name not in other_values, f"The display parameter '{name}' has been defined more than once."
if param.ready(other_values):
if name in app_kwds and param.allow_override:
other_values[name] = app_kwds[name]
@@ -114,7 +114,7 @@ class DynamicDisplayApplicationBuilder:
data_table_name = elem.get('from_data_table', None)
if data_table_name:
data_table = display_application.app.tool_data_tables.get(data_table_name, None)
assert data_table is not None, 'Unable to find data table named "%s".' % data_table_name
assert data_table is not None, f'Unable to find data table named "{data_table_name}".'
assert filename is not None or data_table is not None, 'Filename or data Table is required for dynamic_links.'
skip_startswith = elem.get('skip_startswith', None)
@@ -254,7 +254,7 @@ class PopulatedDisplayApplicationLink:
for name, parameter in self.link.parameters.items():
if parameter.build_url(self.parameters) == url:
return name
raise ValueError("Unknown URL parameter name provided: %s" % url)
raise ValueError(f"Unknown URL parameter name provided: {url}")
@property
def allow_cors(self):
@@ -324,7 +324,7 @@ class DisplayApplication:
elif self._elem:
elem = self._elem
else:
raise Exception("Unable to reload DisplayApplication %s." % (self.name))
raise Exception(f"Unable to reload DisplayApplication {self.name}.")
# All toolshed-specific attributes added by e.g the registry will remain
attr_dict = self._get_attributes_from_elem(elem)
# We will not allow changing the id at this time (we'll need to fix several mappings upstream to handle this case)
@@ -92,7 +92,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter):
return rval
direct_match, target_ext, converted_dataset = data.find_conversion_destination(self.formats)
assert direct_match or target_ext is not None, "No conversion path found for data param: %s" % self.name
assert direct_match or target_ext is not None, f"No conversion path found for data param: {self.name}"
return None
return data
@@ -123,7 +123,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter):
trans.sa_session.add(assoc)
trans.sa_session.flush()
elif converted_dataset and converted_dataset.state == converted_dataset.states.ERROR:
raise Exception("Dataset conversion failed for data parameter: %s" % self.name)
raise Exception(f"Dataset conversion failed for data parameter: {self.name}")
return self.get_value(other_values, dataset_hash, user_hash, trans)
def is_preparing(self, other_values):
@@ -138,7 +138,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter):
if value.state == value.states.OK:
return True
elif value.state == value.states.ERROR:
raise Exception('A data display parameter is in the error state: %s' % (self.name))
raise Exception(f'A data display parameter is in the error state: {self.name}')
return False
@@ -192,7 +192,7 @@ class DisplayParameterValueWrapper:
def url(self):
base_url = self.trans.request.base
if self.parameter.strip_https and base_url[: 5].lower() == 'https':
base_url = "http%s" % base_url[5:]
base_url = f"http{base_url[5:]}"
return "{}{}".format(base_url,
self.trans.app.url_for(controller='dataset',
action="display_application",
+21 -21
View File
@@ -102,18 +102,18 @@ class GenomeGraphs(Tabular):
internal_url = "%s" % app.url_for(controller='dataset',
dataset_id=dataset.id,
action='display_at',
filename='ucsc_' + site_name)
filename=f"ucsc_{site_name}")
display_url = "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type)
display_url = quote_plus(display_url)
# was display_url = quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) )
# redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) )
sl = [f"{site_url}db={dataset.dbkey}", ]
# sl.append("&hgt.customText=%s")
sl.append("&hgGenome_dataSetName={}&hgGenome_dataSetDescription={}".format(dataset.name, 'GalaxyGG_data'))
sl.append(f"&hgGenome_dataSetName={dataset.name}&hgGenome_dataSetDescription=GalaxyGG_data")
sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess")
sl.append("&hgGenome_columnLabels=first row&hgGenome_maxVal=&hgGenome_labelVals=")
sl.append("&hgGenome_doSubmitUpload=submit")
sl.append("&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=%s" % display_url)
sl.append(f"&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile={display_url}")
s = ''.join(sl)
s = quote_plus(s)
redirect_url = s
@@ -130,28 +130,28 @@ class GenomeGraphs(Tabular):
with open(dataset.file_name) as f:
d = f.readlines()[:5]
if len(d) == 0:
out = "Cannot find anything to parse in %s" % dataset.name
out = f"Cannot find anything to parse in {dataset.name}"
return out
hasheader = 0
try:
['%f' % x for x in d[0][1:]] # first is name - see if starts all numerics
[f'{x:f}' for x in d[0][1:]] # first is name - see if starts all numerics
except Exception:
hasheader = 1
# Generate column header
out.append('<tr>')
if hasheader:
for i, name in enumerate(d[0].split()):
out.append('<th>{}.{}</th>'.format(i + 1, name))
out.append(f'<th>{i + 1}.{name}</th>')
d.pop(0)
out.append('</tr>')
for row in d:
out.append('<tr>')
out.append(''.join('<td>%s</td>' % x for x in row.split()))
out.append(''.join(f'<td>{x}</td>' for x in row.split()))
out.append('</tr>')
out.append('</table>')
out = "".join(out)
except Exception as exc:
out = "Can't create peek %s" % exc
out = f"Can't create peek {exc}"
return out
def validate(self, dataset, **kwd):
@@ -284,7 +284,7 @@ class Rgenetics(Html):
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="application/binary">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"application/binary\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="application/binary">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
@@ -324,13 +324,13 @@ class Rgenetics(Html):
efp = dataset.extra_files_path
except Exception:
if verbose:
gal_Log.debug('@@@rgenetics set_meta failed {} - dataset {} has no efp ?'.format(sys.exc_info()[0], dataset.name))
gal_Log.debug(f'@@@rgenetics set_meta failed {sys.exc_info()[0]} - dataset {dataset.name} has no efp ?')
return False
try:
flist = os.listdir(efp)
except Exception:
if verbose:
gal_Log.debug('@@@rgenetics set_meta failed {} - dataset {} has no efp ?'.format(sys.exc_info()[0], dataset.name))
gal_Log.debug(f'@@@rgenetics set_meta failed {sys.exc_info()[0]} - dataset {dataset.name} has no efp ?')
return False
if len(flist) == 0:
if verbose:
@@ -659,7 +659,7 @@ class RexpBase(Html):
for j in range(i + 1, nuse):
kdict = {}
for row in phe: # row is a list of lists
k = '{}{}'.format(row[i], row[j]) # composite key
k = f'{row[i]}{row[j]}' # composite key
kdict[k] = k
if (len(kdict.keys()) == len(concordance[useCols[j]])): # i and j are always matched
delme.append(j)
@@ -707,12 +707,12 @@ class RexpBase(Html):
note that R is weird and does not include the row.name in
the header. why?"""
if not dataset.dataset.purged:
pp = os.path.join(dataset.extra_files_path, '%s.pheno' % dataset.metadata.base_name)
pp = os.path.join(dataset.extra_files_path, f'{dataset.metadata.base_name}.pheno')
try:
with open(pp) as f:
p = f.readlines()
except Exception:
p = ['##failed to find %s' % pp, ]
p = [f'##failed to find {pp}', ]
dataset.peek = ''.join(p[:5])
dataset.blurb = 'Galaxy Rexpression composite file'
else:
@@ -723,12 +723,12 @@ class RexpBase(Html):
"""
expects a .pheno file in the extra_files_dir - ugh
"""
pp = os.path.join(dataset.extra_files_path, '%s.pheno' % dataset.metadata.base_name)
pp = os.path.join(dataset.extra_files_path, f'{dataset.metadata.base_name}.pheno')
try:
with open(pp) as f:
p = f.readlines()
except Exception:
p = ['##failed to find %s' % pp]
p = [f'##failed to find {pp}']
return ''.join(p[:5])
def get_file_peek(self, filename):
@@ -749,7 +749,7 @@ class RexpBase(Html):
"""
bn = dataset.metadata.base_name
flist = os.listdir(dataset.extra_files_path)
rval = ['<html><head><title>Files for Composite Dataset %s</title></head><p/>Comprises the following files:<p/><ul>' % (bn)]
rval = [f'<html><head><title>Files for Composite Dataset {bn}</title></head><p/>Comprises the following files:<p/><ul>']
for fname in flist:
sfname = os.path.split(fname)[-1]
rval.append(f'<li><a href="{sfname}">{sfname}</a>')
@@ -784,7 +784,7 @@ class RexpBase(Html):
if not bn:
bn = '?'
dataset.metadata.base_name = bn
pn = '%s.pheno' % (bn)
pn = f'{bn}.pheno'
pp = os.path.join(dataset.extra_files_path, pn)
dataset.metadata.pheno_path = pp
try:
@@ -824,18 +824,18 @@ class RexpBase(Html):
for i, row in enumerate(p):
lrow = row.strip().split('\t')
if i == 0:
orow = ['<th>%s</th>' % escape(x) for x in lrow]
orow = [f'<th>{escape(x)}</th>' for x in lrow]
orow.insert(0, '<tr>')
orow.append('</tr>')
else:
orow = ['<td>%s</td>' % escape(x) for x in lrow]
orow = [f'<td>{escape(x)}</td>' for x in lrow]
orow.insert(0, '<tr>')
orow.append('</tr>')
out.append(''.join(orow))
out.append('</table>')
out = "\n".join(out)
except Exception as exc:
out = "Can't create html table %s" % unicodify(exc)
out = f"Can't create html table {unicodify(exc)}"
return out
def display_peek(self, dataset):
+1 -1
View File
@@ -40,7 +40,7 @@ class Shapefile(Binary):
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="application/binary">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"application/binary\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="application/binary">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>\n')
+13 -13
View File
@@ -49,7 +49,7 @@ class Image(data.Data):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = 'Image in %s format' % dataset.extension
dataset.peek = f'Image in {dataset.extension} format'
dataset.blurb = nice_size(dataset.get_size())
else:
dataset.peek = 'file does not exist'
@@ -205,24 +205,24 @@ class Pdf(Image):
def create_applet_tag_peek(class_name, archive, params):
text = """
<object classid="java:{}"
text = f"""
<object classid="java:{class_name}"
type="application/x-java-applet"
height="30" width="200" align="center" >
<param name="archive" value="{}"/>""".format(class_name, archive)
<param name="archive" value="{archive}"/>"""
for name, value in params.items():
text += f"""<param name="{name}" value="{value}"/>"""
text += """
text += f"""
<object classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
height="30" width="200" >
<param name="code" value="{}" />
<param name="archive" value="{}"/>""".format(class_name, archive)
<param name="code" value="{class_name}" />
<param name="archive" value="{archive}"/>"""
for name, value in params.items():
text += f"""<param name="{name}" value="{value}"/>"""
text += """<div class="errormessage">You must install and enable Java in your browser in order to access this applet.<div></object>
</object>
"""
return """<div><p align="center">%s</p></div>""" % text
return f"""<div><p align="center">{text}</p></div>"""
@build_sniff_from_prefix
@@ -354,12 +354,12 @@ class Gmaj(data.Data):
if not dataset.dataset.purged:
if hasattr(dataset, 'history_id'):
params = {
"bundle": "display?id=%s&tofile=yes&toext=.zip" % dataset.id,
"bundle": f"display?id={dataset.id}&tofile=yes&toext=.zip",
"buttonlabel": "Launch GMAJ",
"nobutton": "false",
"urlpause": "100",
"debug": "false",
"posturl": "history_add_to?%s" % "&".join("{}={}".format(x[0], quote_plus(str(x[1]))) for x in [('copy_access_from', dataset.id), ('history_id', dataset.history_id), ('ext', 'maf'), ('name', 'GMAJ Output on data %s' % dataset.hid), ('info', 'Added by GMAJ'), ('dbkey', dataset.dbkey)])
"posturl": "history_add_to?%s" % "&".join("{}={}".format(x[0], quote_plus(str(x[1]))) for x in [('copy_access_from', dataset.id), ('history_id', dataset.history_id), ('ext', 'maf'), ('name', f'GMAJ Output on data {dataset.hid}'), ('info', 'Added by GMAJ'), ('dbkey', dataset.dbkey)])
}
class_name = "edu.psu.bx.gmaj.MajApplet.class"
archive = "/static/gmaj/gmaj.jar"
@@ -440,7 +440,7 @@ class Analyze75(Binary):
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"text/plain\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
@@ -600,10 +600,10 @@ class Laj(data.Text):
if not dataset.dataset.purged:
if hasattr(dataset, 'history_id'):
params = {
"alignfile1": "display?id=%s" % dataset.id,
"alignfile1": f"display?id={dataset.id}",
"buttonlabel": "Launch LAJ",
"title": "LAJ in Galaxy",
"posturl": quote_plus("history_add_to?%s" % "&".join(f"{key}={value}" for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())),
"posturl": quote_plus(f"history_add_to?{'&'.join(f'{key}={value}' for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())}"),
"noseq": "true"
}
class_name = "edu.psu.cse.bio.laj.LajApplet.class"
+7 -7
View File
@@ -190,7 +190,7 @@ class Interval(Tabular):
# Make sure we are at the next new line
readline_count = VIEWPORT_MAX_READS_PER_LINE
while line.rstrip('\n\r') == line:
assert readline_count > 0, Exception('Viewport readline count exceeded for dataset %s.' % dataset.id)
assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.')
line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE)
if not line:
break # EOF
@@ -605,7 +605,7 @@ class _RemoteCallMixin:
the data available, followed by redirecting to the remote site with a
link back to the available information.
"""
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename=f'{type}_{site_name}')
internal_url = f"{app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename=f'{type}_{site_name}')}"
base_url = app.config.get("display_at_callback", base_url)
display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" %
(base_url, app.url_for(controller='root'), dataset.id, type))
@@ -734,7 +734,7 @@ class Gff(Tabular, _RemoteCallMixin):
stop = int(elems[1].split('..')[1])
break # use location declared in file
else:
log.debug("line (%s) uses an unsupported ##sequence-region definition." % str(line))
log.debug(f"line ({str(line)}) uses an unsupported ##sequence-region definition.")
# break #no break, if bad definition, we try another line
elif line.startswith("browser position"):
# Allow UCSC style browser and track info in the GFF file
@@ -759,7 +759,7 @@ class Gff(Tabular, _RemoteCallMixin):
# make sure we are at the next new line
readline_count = VIEWPORT_MAX_READS_PER_LINE
while line.rstrip('\n\r') == line:
assert readline_count > 0, Exception('Viewport readline count exceeded for dataset %s.' % dataset.id)
assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.')
line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE)
if not line:
break # EOF
@@ -1139,7 +1139,7 @@ class Wiggle(Tabular, _RemoteCallMixin):
# make sure we are at the next new line
readline_count = VIEWPORT_MAX_READS_PER_LINE
while line.rstrip('\n\r') == line:
assert readline_count > 0, Exception('Viewport readline count exceeded for dataset %s.' % dataset.id)
assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.')
line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE)
if not line:
break # EOF
@@ -1319,7 +1319,7 @@ class CustomTrack(Tabular):
# make sure we are at the next new line
readline_count = VIEWPORT_MAX_READS_PER_LINE
while line.rstrip('\n\r') == line:
assert readline_count > 0, Exception('Viewport readline count exceeded for dataset %s.' % dataset.id)
assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.')
line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE)
if not line:
break # EOF
@@ -1338,7 +1338,7 @@ class CustomTrack(Tabular):
if chrom is not None:
for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey):
if site_name in app.datatypes_registry.get_display_sites('ucsc'):
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name)
internal_url = f"{app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name)}"
display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type))
redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s")
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
+14 -14
View File
@@ -179,11 +179,11 @@ class _Isa(data.Data):
line = line.strip()
if not line:
continue
out.append('<tr><td>%s</td></tr>' % escape(util.unicodify(line, 'utf-8')))
out.append(f"<tr><td>{escape(util.unicodify(line, 'utf-8'))}</td></tr>")
out.append('</table>')
out = "".join(out)
except Exception as exc:
out = "Can't create peek: %s" % util.unicodify(exc)
out = f"Can't create peek: {util.unicodify(exc)}"
return out
# Generate primary file {{{2
@@ -198,7 +198,7 @@ class _Isa(data.Data):
if hasattr(dataset, "extra_files_path"):
rval.append('<div>ISA Dataset composed of the following files:<p/><ul>')
for cmp_file in os.listdir(dataset.extra_files_path):
rval.append('<li><a href="{}" type="text/plain">{}</a></li>'.format(cmp_file, escape(cmp_file)))
rval.append(f'<li><a href="{cmp_file}" type="text/plain">{escape(cmp_file)}</a></li>')
rval.append('</ul></div></html>')
else:
rval.append('<div>ISA Dataset is empty!<p/><ul>')
@@ -269,26 +269,26 @@ class _Isa(data.Data):
# Loop on all studies
for study in investigation.studies:
html += '<h2>Study %s</h2>' % study.identifier
html += '<h3>%s</h3>' % study.title
html += '<p>%s</p>' % study.description
html += '<p>Submitted the %s</p>' % study.submission_date
html += '<p>Released on %s</p>' % study.public_release_date
html += f'<h2>Study {study.identifier}</h2>'
html += f'<h3>{study.title}</h3>'
html += f'<p>{study.description}</p>'
html += f'<p>Submitted the {study.submission_date}</p>'
html += f'<p>Released on {study.public_release_date}</p>'
html += '<p>Experimental factors used: %s</p>' % ', '.join(x.name for x in study.factors)
html += f"<p>Experimental factors used: {', '.join(x.name for x in study.factors)}</p>"
# Loop on all assays of this study
for assay in study.assays:
html += '<h3>Assay %s</h3>' % assay.filename
html += '<p>Measurement type: %s</p>' % assay.measurement_type.term # OntologyAnnotation
html += '<p>Technology type: %s</p>' % assay.technology_type.term # OntologyAnnotation
html += '<p>Technology platform: %s</p>' % assay.technology_platform
html += f'<h3>Assay {assay.filename}</h3>'
html += f'<p>Measurement type: {assay.measurement_type.term}</p>' # OntologyAnnotation
html += f'<p>Technology type: {assay.technology_type.term}</p>' # OntologyAnnotation
html += f'<p>Technology platform: {assay.technology_platform}</p>'
if assay.data_files is not None:
html += '<p>Data files:</p>'
html += '<ul>'
for data_file in assay.data_files:
if data_file.filename != '':
html += '<li>' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
html += f"<li>{escape(util.unicodify(str(data_file.filename), 'utf-8'))} - {escape(util.unicodify(str(data_file.label), 'utf-8'))}</li>"
html += '</ul>'
html += '</body></html>'
+16 -16
View File
@@ -52,7 +52,7 @@ class GenericMolFile(Text):
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
@@ -139,11 +139,11 @@ class SDF(GenericMolFile):
chunk_size = None
if split_params['split_mode'] == 'number_of_parts':
raise Exception('Split mode "%s" is currently not implemented for SD-files.' % split_params['split_mode'])
raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for SD-files.")
elif split_params['split_mode'] == 'to_size':
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
def _read_sdf_records(filename):
lines = []
@@ -222,11 +222,11 @@ class MOL2(GenericMolFile):
chunk_size = None
if split_params['split_mode'] == 'number_of_parts':
raise Exception('Split mode "%s" is currently not implemented for MOL2-files.' % split_params['split_mode'])
raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for MOL2-files.")
elif split_params['split_mode'] == 'to_size':
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
def _read_mol2_records(filename):
lines = []
@@ -307,11 +307,11 @@ class FPS(GenericMolFile):
chunk_size = None
if split_params['split_mode'] == 'number_of_parts':
raise Exception('Split mode "%s" is currently not implemented for MOL2-files.' % split_params['split_mode'])
raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for MOL2-files.")
elif split_params['split_mode'] == 'to_size':
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
def _write_part_fingerprint_file(accumulated_lines):
part_dir = subdir_generator_function()
@@ -667,7 +667,7 @@ class PQR(GenericMolFile):
hetatm_numbers = count_special_lines("^HETATM", dataset.file_name)
chain_ids = ','.join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else 'None'
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "{} atoms and {} HET-atoms\nchain_ids: {}".format(atom_numbers, hetatm_numbers, str(chain_ids))
dataset.blurb = f"{atom_numbers} atoms and {hetatm_numbers} HET-atoms\nchain_ids: {str(chain_ids)}"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -716,7 +716,7 @@ class InChI(Tabular):
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
@@ -761,7 +761,7 @@ class SMILES(Tabular):
if dataset.metadata.number_of_molecules == 1:
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
@@ -822,7 +822,7 @@ class CML(GenericXml):
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
@@ -860,11 +860,11 @@ class CML(GenericXml):
chunk_size = None
if split_params['split_mode'] == 'number_of_parts':
raise Exception('Split mode "%s" is currently not implemented for CML-files.' % split_params['split_mode'])
raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for CML-files.")
elif split_params['split_mode'] == 'to_size':
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
def _read_cml_records(filename):
lines = []
@@ -920,15 +920,15 @@ class CML(GenericXml):
with open(filename) as handle:
header = handle.readline()
if not header:
raise ValueError("CML file %s was empty" % filename)
raise ValueError(f"CML file {filename} was empty")
if not header.lstrip().startswith('<?xml version="1.0"?>'):
out.write(header)
raise ValueError("%s is not a valid XML file!" % filename)
raise ValueError(f"{filename} is not a valid XML file!")
line = handle.readline()
header += line
if not line.lstrip().startswith('<cml xmlns="http://www.xml-cml.org/schema'):
out.write(header)
raise ValueError("%s is not a CML file!" % filename)
raise ValueError(f"{filename} is not a CML file!")
molecule_found = False
for line in handle.readlines():
# We found two required header lines, the next line should start with <molecule >
+3 -3
View File
@@ -334,7 +334,7 @@ class DistanceMatrix(Text):
break
except Exception as e:
if not isinstance(self, PairwiseDistanceMatrix):
log.warning("DistanceMatrix set_meta %s" % e)
log.warning(f"DistanceMatrix set_meta {e}")
@build_sniff_from_prefix
@@ -986,7 +986,7 @@ class SffFlow(Tabular):
flow_values = int(headers[0][0])
dataset.metadata.flow_values = flow_values
except Exception as e:
log.warning("SffFlow set_meta %s" % e)
log.warning(f"SffFlow set_meta {e}")
def make_html_table(self, dataset, skipchars=None):
"""Create HTML table, used for displaying peek"""
@@ -1006,7 +1006,7 @@ class SffFlow(Tabular):
out += self.make_html_peek_rows(dataset, skipchars=skipchars)
out += '</table>'
except Exception as exc:
out = "Can't create peek: %s" % unicodify(exc)
out = f"Can't create peek: {unicodify(exc)}"
return out
+6 -6
View File
@@ -34,7 +34,7 @@ class InfernalCM(Text):
if dataset.metadata.number_of_models == 1:
dataset.blurb = "1 model"
else:
dataset.blurb = "%s models" % dataset.metadata.number_of_models
dataset.blurb = f"{dataset.metadata.number_of_models} models"
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
else:
dataset.peek = 'file does not exist'
@@ -80,7 +80,7 @@ class Hmmer(Text):
try:
return dataset.peek
except Exception:
return "HMMER database (%s)" % (nice_size(dataset.get_size()))
return f"HMMER database ({nice_size(dataset.get_size())})"
@abc.abstractmethod
def sniff_prefix(self, filename):
@@ -153,7 +153,7 @@ class Stockholm_1_0(Text):
if (dataset.metadata.number_of_models == 1):
dataset.blurb = "1 alignment"
else:
dataset.blurb = "%s alignments" % dataset.metadata.number_of_models
dataset.blurb = f"{dataset.metadata.number_of_models} alignments"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
@@ -184,11 +184,11 @@ class Stockholm_1_0(Text):
chunk_size = None
if split_params['split_mode'] == 'number_of_parts':
raise Exception('Split mode "%s" is currently not implemented for STOCKHOLM-files.' % split_params['split_mode'])
raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for STOCKHOLM-files.")
elif split_params['split_mode'] == 'to_size':
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
def _read_stockholm_records(filename):
lines = []
@@ -232,7 +232,7 @@ class MauveXmfa(Text):
if (dataset.metadata.number_of_models == 1):
dataset.blurb = "1 alignment"
else:
dataset.blurb = "%s alignments" % dataset.metadata.number_of_models
dataset.blurb = f"{dataset.metadata.number_of_models} alignments"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
+3 -3
View File
@@ -33,7 +33,7 @@ class BowtieIndex(Html):
"""
bn = dataset.metadata.base_name
flist = os.listdir(dataset.extra_files_path)
rval = ['<html><head><title>Files for Composite Dataset %s</title></head><p/>Comprises the following files:<p/><ul>' % (bn)]
rval = [f'<html><head><title>Files for Composite Dataset {bn}</title></head><p/>Comprises the following files:<p/><ul>']
for fname in flist:
sfname = os.path.split(fname)[-1]
rval.append(f'<li><a href="{sfname}">{sfname}</a>')
@@ -44,8 +44,8 @@ class BowtieIndex(Html):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "Bowtie index file (%s)" % (dataset.metadata.sequence_space)
dataset.blurb = "%s space" % (dataset.metadata.sequence_space)
dataset.peek = f"Bowtie index file ({dataset.metadata.sequence_space})"
dataset.blurb = f"{dataset.metadata.sequence_space} space"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
+1 -1
View File
@@ -39,7 +39,7 @@ class Phylip(Text):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if dataset.metadata.sequences:
dataset.blurb = "%s sequences" % util.commaify(str(dataset.metadata.sequences))
dataset.blurb = f"{util.commaify(str(dataset.metadata.sequences))} sequences"
else:
dataset.blurb = nice_size(dataset.get_size())
else:
+3 -3
View File
@@ -18,7 +18,7 @@ class Smat(Text):
try:
return dataset.peek
except Exception:
return "ESTScan scores matrices (%s)" % (nice_size(dataset.get_size()))
return f"ESTScan scores matrices ({nice_size(dataset.get_size())})"
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
@@ -103,7 +103,7 @@ class PlantTribesKsComponents(Tabular):
try:
return dataset.peek
except Exception:
return "Significant components in the Ks distribution (%s)" % (nice_size(dataset.get_size()))
return f"Significant components in the Ks distribution ({nice_size(dataset.get_size())})"
def set_meta(self, dataset, **kwd):
"""
@@ -133,7 +133,7 @@ class PlantTribesKsComponents(Tabular):
if (dataset.metadata.number_comp == 1):
dataset.blurb = "1 significant component"
else:
dataset.blurb = "%s significant components" % dataset.metadata.number_comp
dataset.blurb = f"{dataset.metadata.number_comp} significant components"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
+5 -5
View File
@@ -46,7 +46,7 @@ class Wiff(Binary):
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"text/plain\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
@@ -835,7 +835,7 @@ class ThermoRAW(Binary):
try:
return dataset.peek
except Exception:
return "Thermo Finnigan RAW file (%s)" % (nice_size(dataset.get_size()))
return f"Thermo Finnigan RAW file ({nice_size(dataset.get_size())})"
@build_sniff_from_prefix
@@ -898,7 +898,7 @@ class SPLib(Msp):
if composite_file.optional:
opt_text = ' (optional)'
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"text/plain\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
@@ -942,7 +942,7 @@ class Ms2(Text):
for header_field in ['CreationDate', 'Extractor', 'ExtractorVersion', 'ExtractorOptions']:
found_header = False
for header_line in header_lines:
if header_line.startswith('H\t%s' % (header_field)):
if header_line.startswith(f'H\t{header_field}'):
found_header = True
break
if not found_header:
@@ -993,7 +993,7 @@ class ImzML(Binary):
fn = composite_name
opt_text = ''
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
rval.append(f"<li><a href=\"{fn}\" type=\"text/plain\">{fn} ({composite_file.get('description')})</a>{opt_text}</li>")
else:
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
+19 -19
View File
@@ -112,9 +112,9 @@ class Registry:
root = tree.getroot()
# Load datatypes and converters from config
if deactivate:
self.log.debug('Deactivating datatypes from %s' % config)
self.log.debug(f'Deactivating datatypes from {config}')
else:
self.log.debug('Loading datatypes from %s' % config)
self.log.debug(f'Loading datatypes from {config}')
else:
root = config
registration = root.find('registration')
@@ -131,7 +131,7 @@ class Registry:
self.converters_path_attr = os.path.abspath(os.path.join(os.path.dirname(__file__), 'converters'))
self.converters_path = self.converters_path_attr
if not os.path.isdir(self.converters_path):
raise ConfigurationError("Directory does not exist: %s" % self.converters_path)
raise ConfigurationError(f"Directory does not exist: {self.converters_path}")
if use_display_applications:
if not self.display_applications_path:
self.display_path_attr = registration.get('display_path', 'display_applications')
@@ -204,7 +204,7 @@ class Registry:
del self.datatypes_by_extension[extension]
if extension in self.upload_file_formats:
self.upload_file_formats.remove(extension)
self.log.debug("Removed datatype with extension '%s' from the registry." % extension)
self.log.debug(f"Removed datatype with extension '{extension}' from the registry.")
else:
# We are loading new datatype, so we'll make sure it is correctly defined before proceeding.
can_process_datatype = False
@@ -247,14 +247,14 @@ class Registry:
for mod in fields:
module = getattr(module, mod)
datatype_class = getattr(module, datatype_class_name)
self.log.debug('Retrieved datatype module {}:{} from the datatype registry for extension {}.'.format(str(datatype_module), datatype_class_name, extension))
self.log.debug(f'Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}.')
except Exception:
self.log.exception('Error importing datatype module %s', str(datatype_module))
ok = False
elif type_extension is not None:
try:
datatype_class = self.datatypes_by_extension[type_extension].__class__
self.log.debug('Retrieved datatype module {} from type_extension {} for extension {}.'.format(str(datatype_class.__name__), type_extension, extension))
self.log.debug(f'Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}.')
except Exception:
self.log.exception('Error determining datatype_class for type_extension %s', str(type_extension))
ok = False
@@ -307,7 +307,7 @@ class Registry:
for composite_file in elem.findall('composite_file'):
name = composite_file.get('name', None)
if name is None:
self.log.warning("You must provide a name for your composite_file (%s)." % composite_file)
self.log.warning(f"You must provide a name for your composite_file ({composite_file}).")
optional = composite_file.get('optional', False)
mimetype = composite_file.get('mimetype', None)
self.datatypes_by_extension[extension].add_composite_file(name, optional=optional, mimetype=mimetype)
@@ -339,7 +339,7 @@ class Registry:
elif auto_compressed_type == "bz2":
dynamic_parent = binary.Bz2DynamicCompressedArchive
else:
raise Exception("Unknown auto compression type [%s]" % auto_compressed_type)
raise Exception(f"Unknown auto compression type [{auto_compressed_type}]")
attributes["file_ext"] = compressed_extension
attributes["uncompressed_datatype_instance"] = datatype_instance
compressed_datatype_class = type(auto_compressed_type_name, (datatype_class, dynamic_parent, ), attributes)
@@ -357,7 +357,7 @@ class Registry:
"description": description,
"description_url": description_url,
})
self.converters.append(("%s_to_uncompressed.xml" % auto_compressed_type, compressed_extension, extension))
self.converters.append((f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension))
if datatype_class not in compressed_sniffers:
compressed_sniffers[datatype_class] = []
if sniff_compressed_types:
@@ -411,7 +411,7 @@ class Registry:
site_type = build_site_config.get('type')
path = build_site_config.get('file')
if not os.path.exists(path):
sample_path = "%s.sample" % path
sample_path = f"{path}.sample"
if os.path.exists(sample_path):
self.log.debug(f"Build site file [{path}] not found using sample [{sample_path}].")
path = sample_path
@@ -511,12 +511,12 @@ class Registry:
if sniffer_class == s_e_c:
del self.sniffer_elems[index]
sniffer_elem_classes = [elem.attrib['type'] for elem in self.sniffer_elems]
self.log.debug("Removed sniffer element for datatype '%s'" % str(dtype))
self.log.debug(f"Removed sniffer element for datatype '{str(dtype)}'")
break
for sniffer_class in self.sniff_order:
if sniffer_class.__class__ == aclass.__class__:
self.sniff_order.remove(sniffer_class)
self.log.debug("Removed sniffer class for datatype '%s' from sniff order" % str(dtype))
self.log.debug(f"Removed sniffer class for datatype '{str(dtype)}' from sniff order")
break
else:
# We are loading new sniffer, so see if we have a conflicting sniffer already loaded.
@@ -527,14 +527,14 @@ class Registry:
conflict = True
if override:
del self.sniff_order[conflict_loc]
self.log.debug("Removed conflicting sniffer for datatype '%s'" % dtype)
self.log.debug(f"Removed conflicting sniffer for datatype '{dtype}'")
break
if not conflict or override:
if compressed_sniffers and aclass.__class__ in compressed_sniffers:
for compressed_sniffer in compressed_sniffers[aclass.__class__]:
self.sniff_order.append(compressed_sniffer)
self.sniff_order.append(aclass)
self.log.debug("Loaded sniffer for datatype '%s'" % dtype)
self.log.debug(f"Loaded sniffer for datatype '{dtype}'")
# Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary.
sniffer_class = elem.get('type', None)
if sniffer_class is not None:
@@ -573,7 +573,7 @@ class Registry:
except KeyError:
# datatype was never declared
mimetype = default
self.log.warning('unknown mimetype in data factory %s' % str(ext))
self.log.warning(f'unknown mimetype in data factory {str(ext)}')
return mimetype
def get_datatype_by_extension(self, ext):
@@ -644,9 +644,9 @@ class Registry:
self.log.debug("Loaded converter: %s", converter.id)
except Exception:
if deactivate:
self.log.exception("Error deactivating converter from (%s)" % converter_path)
self.log.exception(f"Error deactivating converter from ({converter_path})")
else:
self.log.exception("Error loading converter (%s)" % converter_path)
self.log.exception(f"Error loading converter ({converter_path})")
def load_display_applications(self, app, installed_repository_dict=None, deactivate=False):
"""
@@ -712,9 +712,9 @@ class Registry:
self.log.debug(f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}.")
except Exception:
if deactivate:
self.log.exception("Error deactivating display application (%s)" % config_path)
self.log.exception(f"Error deactivating display application ({config_path})")
else:
self.log.exception("Error loading display application (%s)" % config_path)
self.log.exception(f"Error loading display application ({config_path})")
# Handle display_application subclass inheritance.
for extension, d_type1 in self.datatypes_by_extension.items():
for d_type2, display_app in self.inherit_display_application_by_class:
+15 -15
View File
@@ -113,7 +113,7 @@ class Sequence(data.Text):
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.sequences:
dataset.blurb = "%s sequences" % util.commaify(str(dataset.metadata.sequences))
dataset.blurb = f"{util.commaify(str(dataset.metadata.sequences))} sequences"
else:
dataset.blurb = nice_size(dataset.get_size())
else:
@@ -137,7 +137,7 @@ class Sequence(data.Text):
if rem > 0:
sequences_per_file.append(rem)
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
return sequences_per_file
@classmethod
@@ -192,7 +192,7 @@ class Sequence(data.Text):
if toc_file_datasets is not None:
toc = toc_file_datasets[ds_no]
split_data['args']['toc_file'] = toc.file_name
with open(os.path.join(dir, 'split_info_%s.json' % base_name), 'w') as f:
with open(os.path.join(dir, f'split_info_{base_name}.json'), 'w') as f:
json.dump(split_data, f)
start_sequence += sequences_per_file[part_no]
return directories
@@ -232,7 +232,7 @@ class Sequence(data.Text):
current_sequence += int(sections[i]['sequences'])
i += 1
if i == len(sections): # bad input data!
raise Exception('No FQTOC section contains starting sequence %s' % start_sequence)
raise Exception(f'No FQTOC section contains starting sequence {start_sequence}')
# These two variables act as an accumulator for consecutive entire blocks that
# can be copied verbatim (without decompressing)
@@ -268,7 +268,7 @@ class Sequence(data.Text):
result.append(copy_chunk_cmd % (start_chunk, end_chunk - start_chunk, input_name, output_name))
if sequence_count > 0:
raise Exception('%s sequences not found in file' % sequence_count)
raise Exception(f'{sequence_count} sequences not found in file')
return result
@@ -285,10 +285,10 @@ class Sequence(data.Text):
line_count = sequence_count * 4
# TODO: verify that tail can handle 64-bit numbers
if is_compressed:
cmd = 'zcat "{}" | ( tail -n +{} 2> /dev/null) | head -{} | gzip -c'.format(input_name, start_line + 1, line_count)
cmd = f'zcat "{input_name}" | ( tail -n +{start_line + 1} 2> /dev/null) | head -{line_count} | gzip -c'
else:
cmd = 'tail -n +{} "{}" 2> /dev/null | head -{}'.format(start_line + 1, input_name, line_count)
cmd += ' > "%s"' % output_name
cmd = f'tail -n +{start_line + 1} "{input_name}" 2> /dev/null | head -{line_count}'
cmd += f' > "{output_name}"'
return [cmd]
@@ -412,7 +412,7 @@ class Fasta(Sequence):
log.debug("Split %s into batches of %i records..." % (input_file, batch_size))
cls._count_split(input_file, batch_size, subdir_generator_function)
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
raise Exception(f"Unsupported split mode {split_params['split_mode']}")
@classmethod
def _size_split(cls, input_file, chunk_size, subdir_generator_function):
@@ -625,10 +625,10 @@ class Fastg(Sequence):
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.sequences:
dataset.blurb = "%s sequences" % util.commaify(str(dataset.metadata.sequences))
dataset.blurb = f"{util.commaify(str(dataset.metadata.sequences))} sequences"
else:
dataset.blurb = nice_size(dataset.get_size())
dataset.blurb += '\nversion=%s' % dataset.metadata.version
dataset.blurb += f'\nversion={dataset.metadata.version}'
for k, v in dataset.metadata.properties.items():
if k != 'version':
dataset.blurb += f'\n{k}={v}'
@@ -908,7 +908,7 @@ class Maf(Alignment):
# The file must exist on disk for the get_file_peek() method
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.blocks:
dataset.blurb = "%s blocks" % util.commaify(str(dataset.metadata.blocks))
dataset.blurb = f"{util.commaify(str(dataset.metadata.blocks))} blocks"
else:
# Number of blocks is not known ( this should not happen ), and auto-detect is
# needed to set metadata
@@ -928,7 +928,7 @@ class Maf(Alignment):
try:
out.append('<tr><th>Species:&nbsp;')
for species in dataset.metadata.species:
out.append('%s&nbsp;' % species)
out.append(f'{species}&nbsp;')
out.append('</th></tr>')
if not dataset.peek:
dataset.set_peek()
@@ -938,11 +938,11 @@ class Maf(Alignment):
line = line.strip()
if not line:
continue
out.append('<tr><td>%s</td></tr>' % escape(line))
out.append(f'<tr><td>{escape(line)}</td></tr>')
out.append('</table>')
out = "".join(out)
except Exception as exc:
out = "Can't create peek %s" % exc
out = f"Can't create peek {exc}"
return out
def sniff_prefix(self, file_prefix):
+1 -1
View File
@@ -710,7 +710,7 @@ def handle_compressed_file(
except OSError as e:
os.remove(uncompressed.name)
compressed_file.close()
raise OSError('Problem uncompressing {} data, please try retrieving the data uncompressed: {}'.format(compressed_type, util.unicodify(e)))
raise OSError(f'Problem uncompressing {compressed_type} data, please try retrieving the data uncompressed: {util.unicodify(e)}')
if not chunk:
break
uncompressed.write(chunk)
+19 -19
View File
@@ -55,7 +55,7 @@ class TabularData(data.Text):
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=False, **kwd):
super().set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap)
if dataset.metadata.comment_lines:
dataset.blurb = "{}, {} comments".format(dataset.blurb, util.commaify(str(dataset.metadata.comment_lines)))
dataset.blurb = f"{dataset.blurb}, {util.commaify(str(dataset.metadata.comment_lines))} comments"
def displayable(self, dataset):
try:
@@ -137,7 +137,7 @@ class TabularData(data.Text):
out.append('</table>')
out = "".join(out)
except Exception as exc:
out = "Can't create peek: %s" % util.unicodify(exc)
out = f"Can't create peek: {util.unicodify(exc)}"
return out
def make_html_peek_header(self, dataset, skipchars=None, column_names=None, column_number_format='%s', column_parameter_alias=None, **kwargs):
@@ -179,12 +179,12 @@ class TabularData(data.Text):
if header is None:
out.append(column_number_format % str(i + 1))
else:
out.append('{}.{}'.format(str(i + 1), escape(header)))
out.append(f'{str(i + 1)}.{escape(header)}')
out.append('</th>')
out.append('</tr>')
except Exception as exc:
log.exception('make_html_peek_header failed on HDA %s', dataset.id)
raise Exception("Can't create peek header: %s" % util.unicodify(exc))
raise Exception(f"Can't create peek header: {util.unicodify(exc)}")
return "".join(out)
def make_html_peek_rows(self, dataset, skipchars=None, **kwargs):
@@ -204,7 +204,7 @@ class TabularData(data.Text):
for i, line in enumerate(peek.splitlines()):
if i >= self.data_line_offset:
if line.startswith(tuple(skipchars)):
out.append('<tr><td colspan="100%%">%s</td></tr>' % escape(line))
out.append(f'<tr><td colspan="100%">{escape(line)}</td></tr>')
elif line:
elems = line.split(dataset.metadata.delimiter)
elems = elems[:min(len(elems), self.max_peek_columns)]
@@ -213,15 +213,15 @@ class TabularData(data.Text):
elems.extend([''] * (columns - len(elems)))
# we may have an invalid comment line or invalid data
if len(elems) != columns:
out.append('<tr><td colspan="100%%">%s</td></tr>' % escape(line))
out.append(f'<tr><td colspan="100%">{escape(line)}</td></tr>')
else:
out.append('<tr>')
for elem in elems:
out.append('<td>%s</td>' % escape(elem))
out.append(f'<td>{escape(elem)}</td>')
out.append('</tr>')
except Exception as exc:
log.exception('make_html_peek_rows failed on HDA %s', dataset.id)
raise Exception("Can't create peek rows: %s" % util.unicodify(exc))
raise Exception(f"Can't create peek rows: {util.unicodify(exc)}")
return "".join(out)
def display_peek(self, dataset):
@@ -344,7 +344,7 @@ class Tabular(TabularData):
is_column_type = {} # Dict to store column type string to checking function
for column_type in column_type_set_order:
is_column_type[column_type] = locals()["is_%s" % (column_type)]
is_column_type[column_type] = locals()[f"is_{column_type}"]
def guess_column_type(column_text):
for column_type in column_type_set_order:
@@ -764,13 +764,13 @@ class BaseVcf(Tabular):
stderr_f = tempfile.NamedTemporaryFile(prefix="bam_merge_stderr")
stderr_name = stderr_f.name
command = ["bcftools", "concat"] + split_files + ["-o", output_file]
log.info("Merging vcf files with command [%s]" % " ".join(command))
log.info(f"Merging vcf files with command [{' '.join(command)}]")
exit_code = subprocess.call(args=command, stderr=open(stderr_name, 'wb'))
with open(stderr_name, "rb") as f:
stderr = f.read().strip()
# Did merge succeed?
if exit_code != 0:
raise Exception("Error merging VCF files: %s" % stderr)
raise Exception(f"Error merging VCF files: {stderr}")
def validate(self, dataset, **kwd):
def validate_row(row):
@@ -829,7 +829,7 @@ class VcfGz(BaseVcf, binary.Binary):
try:
pysam.tabix_index(dataset.file_name, index=index_file.file_name, preset='vcf', keep_original=True, force=True)
except Exception as e:
raise Exception('Error setting VCF.gz metadata: %s' % (util.unicodify(e)))
raise Exception(f'Error setting VCF.gz metadata: {util.unicodify(e)}')
dataset.metadata.tabix_index = index_file
@@ -865,17 +865,17 @@ class Eland(Tabular):
# Generate column header
out.append('<tr>')
for i, name in enumerate(self.column_names):
out.append('<th>{}.{}</th>'.format(str(i + 1), name))
out.append(f'<th>{str(i + 1)}.{name}</th>')
# This data type requires at least 11 columns in the data
if dataset.metadata.columns - len(self.column_names) > 0:
for i in range(len(self.column_names), max(dataset.metadata.columns, self.max_peek_columns)):
out.append('<th>%s</th>' % str(i + 1))
out.append(f'<th>{str(i + 1)}</th>')
out.append('</tr>')
out.append(self.make_html_peek_rows(dataset, skipchars=skipchars, peek=peek))
out.append('</table>')
out = "".join(out)
except Exception as exc:
out = "Can't create peek %s" % exc
out = f"Can't create peek {exc}"
return out
def sniff_prefix(self, file_prefix):
@@ -1132,8 +1132,8 @@ class ConnectivityTable(Tabular):
edam_format = "format_3309"
file_ext = "ct"
header_regexp = re.compile("^[0-9]+" + "(?:\t|[ ]+)" + ".*?" + "(?:ENERGY|energy|dG)" + "[ \t].*?=")
structure_regexp = re.compile("^[0-9]+" + "(?:\t|[ ]+)" + "[ACGTURYKMSWBDHVN]+" + "(?:\t|[ ]+)" + "[^\t]+" + "(?:\t|[ ]+)" + "[^\t]+" + "(?:\t|[ ]+)" + "[^\t]+" + "(?:\t|[ ]+)" + "[^\t]+")
header_regexp = re.compile("^[0-9]+(?: |[ ]+).*?(?:ENERGY|energy|dG)[ ].*?=")
structure_regexp = re.compile("^[0-9]+(?: |[ ]+)[ACGTURYKMSWBDHVN]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+")
def __init__(self, **kwd):
super().__init__(**kwd)
@@ -1201,7 +1201,7 @@ class ConnectivityTable(Tabular):
else:
if j != int(re.split(r'\W+', line, 1)[0]):
return False
elif j == length: # Last line of first sequence has been recheached
elif j == length: # Last line of first sequence has been reached
return True
else:
j += 1
@@ -1230,7 +1230,7 @@ class ConnectivityTable(Tabular):
ck_data_body = re.sub('\n[ \t]+', '\n', ck_data_body)
ck_data_body = re.sub('[ ]+', '\t', ck_data_body)
return dumps({'ck_data': util.unicodify(ck_data_header + "\n" + ck_data_body), 'ck_index': ck_index + 1})
return dumps({'ck_data': util.unicodify(f"{ck_data_header}\n{ck_data_body}"), 'ck_index': ck_index + 1})
@build_sniff_from_prefix
+9 -9
View File
@@ -105,7 +105,7 @@ class Json(Text):
try:
return dataset.peek
except Exception:
return "JSON file (%s)" % (nice_size(dataset.get_size()))
return f"JSON file ({nice_size(dataset.get_size())})"
class ExpressionJson(Json):
@@ -183,7 +183,7 @@ class Ipynb(Json):
try:
cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename]
subprocess.check_call(cmd)
ofilename = '%s.html' % ofilename
ofilename = f'{ofilename}.html'
except subprocess.CalledProcessError:
ofilename = dataset.file_name
log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex.quote, cmd)))
@@ -617,12 +617,12 @@ class SnpEffDb(Text):
dataset.metadata.annotation = annotations
try:
with open(dataset.file_name, 'w') as fh:
fh.write("%s\n" % genome_version if genome_version else 'Genome unknown')
fh.write("%s\n" % snpeff_version if snpeff_version else 'SnpEff version unknown')
fh.write(f"{genome_version}\n" if genome_version else 'Genome unknown')
fh.write(f"{snpeff_version}\n" if snpeff_version else 'SnpEff version unknown')
if annotations:
fh.write("annotations: %s\n" % ','.join(annotations))
fh.write(f"annotations: {','.join(annotations)}\n")
if regulations:
fh.write("regulations: %s\n" % ','.join(regulations))
fh.write(f"regulations: {','.join(regulations)}\n")
except Exception:
pass
@@ -663,7 +663,7 @@ class SnpSiftDbNSFP(Text):
"""
cannot do this until we are setting metadata
"""
annotations = "dbNSFP Annotations: %s\n" % ','.join(dataset.metadata.annotation)
annotations = f"dbNSFP Annotations: {','.join(dataset.metadata.annotation)}\n"
with open(dataset.file_name, 'a') as f:
if dataset.metadata.bgzip:
bn = dataset.metadata.bgzip
@@ -695,8 +695,8 @@ class SnpSiftDbNSFP(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = '{} : {}'.format(dataset.metadata.reference_name, ','.join(dataset.metadata.annotation))
dataset.blurb = '%s' % dataset.metadata.reference_name
dataset.peek = f"{dataset.metadata.reference_name} : {','.join(dataset.metadata.annotation)}"
dataset.blurb = f'{dataset.metadata.reference_name}'
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
+1 -1
View File
@@ -130,7 +130,7 @@ class Rdf(xml.GenericXml, Triples):
def sniff_prefix(self, file_prefix):
# <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" ...
match = re.compile(r'xmlns:([^=]*)="http://www.w3.org/1999/02/22-rdf-syntax-ns#"').search(file_prefix.contents_header)
if not match and (match.group(1) + ":RDF") in file_prefix.contents_header:
if not match and (f"{match.group(1)}:RDF") in file_prefix.contents_header:
return True
return False
+3 -3
View File
@@ -371,12 +371,12 @@ def parse_gff3_attributes(attr_str):
for tag_value_pair in attributes_list:
pair = tag_value_pair.strip().split("=")
if len(pair) == 1:
raise Exception("Attribute '%s' does not contain a '='" % tag_value_pair)
raise Exception(f"Attribute '{tag_value_pair}' does not contain a '='")
if pair == '':
continue
tag = pair[0].strip()
if tag == '':
raise Exception("Empty tag in attribute '%s'" % tag_value_pair)
raise Exception(f"Empty tag in attribute '{tag_value_pair}'")
value = pair[1].strip()
attributes[tag] = value
return attributes
@@ -424,7 +424,7 @@ def read_unordered_gtf(iterator, strict=False):
# transcripts with same ID on different chromosomes; this occurs in some popular
# datasources, such as RefGenes in UCSC.
def key_fn(fields):
return fields[0] + '_' + get_transcript_id(fields)
return f"{fields[0]}_{get_transcript_id(fields)}"
# Aggregate intervals by transcript_id and collect comments.
feature_intervals = {}
+3 -3
View File
@@ -51,7 +51,7 @@ def get_species_in_block(block):
def tool_fail(msg="Unknown Error"):
sys.exit("Fatal Error: %s" % msg)
sys.exit(f"Fatal Error: {msg}")
class TempFileHandler:
@@ -604,7 +604,7 @@ def get_starts_ends_fields_from_gene_bed(line):
fields = line.split()
# Requires atleast 12 BED columns
if len(fields) < 12:
raise Exception("Not a proper 12 column BED line (%s)." % line)
raise Exception(f"Not a proper 12 column BED line ({line}).")
tx_start = int(fields[1])
strand = fields[5]
if strand != '-':
@@ -688,7 +688,7 @@ def get_fasta_header(component, attributes=None, suffix=None):
if suffix:
header = f"{header}{suffix}"
else:
header = "{}{}".format(header, src_split(component.src)[0])
header = f"{header}{src_split(component.src)[0]}"
return header
+1 -1
View File
@@ -161,7 +161,7 @@ class ConditionalDependencies:
def check(self, name):
try:
name = name.replace('-', '_').replace('.', '_')
return getattr(self, 'check_' + name)()
return getattr(self, f"check_{name}")()
except Exception:
return False
+3 -3
View File
@@ -77,10 +77,10 @@ class ConfiguredFileSources:
def get_file_source_path(self, uri):
"""Parse uri into a FileSource object and a path relative to its base."""
if "://" not in uri:
raise exceptions.RequestParameterInvalidException("Invalid uri [%s]" % uri)
raise exceptions.RequestParameterInvalidException(f"Invalid uri [{uri}]")
scheme, rest = uri.split("://", 1)
if scheme not in self.get_schemes():
raise exceptions.RequestParameterInvalidException("Unsupported URI scheme [%s]" % scheme)
raise exceptions.RequestParameterInvalidException(f"Unsupported URI scheme [{scheme}]")
if scheme != "gxfiles":
# prefix unused
@@ -132,7 +132,7 @@ class ConfiguredFileSources:
# is this string a URI this object understands how to realize
if path_or_uri.startswith("gx") and "://" in path_or_uri:
for scheme in self.get_schemes():
if path_or_uri.startswith("%s://" % scheme):
if path_or_uri.startswith(f"{scheme}://"):
return True
return False
+2 -2
View File
@@ -83,7 +83,7 @@ class BaseFilesSource(FilesSource):
def get_uri_root(self):
prefix = self.get_prefix()
scheme = self.get_scheme()
root = "%s://" % scheme
root = f"{scheme}://"
if prefix:
root = uri_join(root, prefix)
return root
@@ -213,7 +213,7 @@ def uri_join(*args):
arg0 = args[0]
if "://" in arg0:
scheme, path = arg0.split("://", 1)
rval = scheme + "://" + (slash_join(path, *args[1:]) if path else slash_join(*args[1:]))
rval = f"{scheme}://{slash_join(path, *args[1:]) if path else slash_join(*args[1:])}"
else:
rval = slash_join(*args)
return rval
+2 -2
View File
@@ -33,7 +33,7 @@ class PosixFilesSource(BaseFilesSource):
def _list(self, path="/", recursive=True, user_context=None):
dir_path = self._to_native_path(path, user_context=user_context)
if not self._safe_directory(dir_path):
raise exceptions.ObjectNotFound('The specified directory does not exist [%s].' % dir_path)
raise exceptions.ObjectNotFound(f'The specified directory does not exist [{dir_path}].')
if recursive:
res = []
for (p, dirs, files) in safe_walk(dir_path, allowlist=self._allowlist):
@@ -107,7 +107,7 @@ class PosixFilesSource(BaseFilesSource):
def _safe_directory(self, directory):
if self.enforce_symlink_security:
if not safe_path(directory, allowlist=self._allowlist):
raise exceptions.ConfigDoesNotAllowException('directory (%s) is a symlink to a location not on the allowlist' % directory)
raise exceptions.ConfigDoesNotAllowException(f'directory ({directory}) is a symlink to a location not on the allowlist')
if not os.path.exists(directory):
return False
+1 -1
View File
@@ -62,7 +62,7 @@ class FormDefinitionFactory:
if fields_elem is not None:
for field_elem in fields_elem.findall('field'):
field_type = field_elem.get('type')
assert field_type in self.field_type_factories, 'Invalid form field type ( %s ).' % field_type
assert field_type in self.field_type_factories, f'Invalid form field type ( {field_type} ).'
fields.append(self.field_type_factories[field_type].from_elem(field_elem, layout))
# create and return new form
return self.new(form_type, name, description=description, fields=fields, layout=layout, form_definition_current=form_definition_current)
+1 -1
View File
@@ -98,7 +98,7 @@ class OutputsToWorkingDirectoryPathRewriter:
if self.outputs_directory_name is not None:
base_output_directory = os.path.join(base_output_directory, self.outputs_directory_name)
# set false_path to uuid, no harm even if object store uses id
false_path = os.path.join(base_output_directory, "galaxy_dataset_%s.dat" % dataset.dataset.uuid)
false_path = os.path.join(base_output_directory, f"galaxy_dataset_{dataset.dataset.uuid}.dat")
return false_path
else:
return None
+2 -2
View File
@@ -401,7 +401,7 @@ def collect_primary_datasets(job_context, output, input_ext):
fields_match = discovered_file.match
if not fields_match:
# Before I guess pop() would just have thrown an IndexError
raise Exception("Problem parsing metadata fields for file %s" % filename)
raise Exception(f"Problem parsing metadata fields for file {filename}")
designation = fields_match.designation
ext = fields_match.ext
if ext == "input":
@@ -610,7 +610,7 @@ def read_exit_code_from(exit_code_file, id_tag):
def default_exit_code_file(files_dir, id_tag):
return os.path.join(files_dir, 'galaxy_%s.ec' % id_tag)
return os.path.join(files_dir, f'galaxy_{id_tag}.ec')
def collect_extra_files(object_store, dataset, job_working_directory):
+1 -1
View File
@@ -25,7 +25,7 @@ class JobPortsView:
def __authorize_job_access(self, encoded_job_id, **kwargs):
key = "job_key"
if key not in kwargs:
error_message = "Job files action requires a valid '%s'." % key
error_message = f"Job files action requires a valid '{key}'."
raise ObjectAttributeMissingException(error_message)
job_id = self._security.decode_id(encoded_job_id)
+11 -11
View File
@@ -69,30 +69,30 @@ class CollectlCli:
if self.mode == MODE_RECORD:
mode_arg = ""
elif self.mode == MODE_PLAYBACK:
mode_arg = "-P -p '%s'" % playback_path
mode_arg = f"-P -p '{playback_path}'"
else:
raise Exception("Invalid mode supplied to CollectlCli - %s" % self.mode)
raise Exception(f"Invalid mode supplied to CollectlCli - {self.mode}")
command_args["mode_arg"] = mode_arg
command_args["interval_arg"] = self.__interval_arg(kwargs)
destination = kwargs.get("destination_path", None)
if destination:
destination_arg = "-f '%s'" % destination
destination_arg = f"-f '{destination}'"
else:
destination_arg = ""
command_args["destination_arg"] = destination_arg
procfilt = kwargs.get("procfilt", None)
command_args["procfilt_arg"] = "" if not procfilt else "--procfilt %s" % procfilt
command_args["procfilt_arg"] = "" if not procfilt else f"--procfilt {procfilt}"
command_args["subsystems_arg"] = self.__subsystems_arg(kwargs.get("subsystems", []))
flush = kwargs.get("flush", None)
command_args["flush_arg"] = "--flush %s" % flush if flush else ""
command_args["flush_arg"] = f"--flush {flush}" if flush else ""
sep = kwargs.get("sep", None)
command_args["sep_arg"] = "--sep=%s" % sep if sep else ""
command_args["sep_arg"] = f"--sep={sep}" if sep else ""
self.command_args = command_args
def __subsystems_arg(self, subsystems):
if subsystems:
return "-s%s" % "".join(s.command_line_arg for s in subsystems)
return f"-s{''.join(s.command_line_arg for s in subsystems)}"
else:
return ""
@@ -105,7 +105,7 @@ class CollectlCli:
return ""
self.__validate_interval_arg(interval)
interval_arg = "-i %s" % interval
interval_arg = f"-i {interval}"
interval2 = kwargs.get("interval2", None)
if not interval2:
return interval_arg
@@ -121,17 +121,17 @@ class CollectlCli:
def __validate_interval_arg(self, value, multiple_of=None):
if value and not str(value).isdigit():
raise Exception("Invalid interval argument supplied, must be integer %s" % value)
raise Exception(f"Invalid interval argument supplied, must be integer {value}")
if multiple_of:
if int(value) % multiple_of != 0:
raise Exception("Invalid interval argument supplied, must multiple of %s" % multiple_of)
raise Exception(f"Invalid interval argument supplied, must multiple of {multiple_of}")
def build_command_line(self):
return COMMAND_LINE_TEMPLATE.substitute(**self.command_args)
def run(self, stdout=subprocess.PIPE, stderr=subprocess.PIPE):
command_line = self.build_command_line()
log.info("Executing %s" % command_line)
log.info(f"Executing {command_line}")
proc = subprocess.Popen(command_line, shell=True, stdout=stdout, stderr=stderr)
return_code = proc.wait()
if return_code:
+3 -3
View File
@@ -96,9 +96,9 @@ def parse_process_statistics(statistics):
# Check for validity...
for statistic in statistics:
if statistic[0] not in STATISTIC_TYPES:
raise Exception("Unknown statistic type encountered %s" % statistic[0])
raise Exception(f"Unknown statistic type encountered {statistic[0]}")
if statistic[1] not in PROCESS_COLUMNS:
raise Exception("Unknown process column encountered %s" % statistic[1])
raise Exception(f"Unknown process column encountered {statistic[1]}")
return statistics
@@ -119,7 +119,7 @@ def _read_process_statistics(tsv_file, pid, statistics):
if current_interval is None:
for header, expected_header in zip(row, PROCESS_COLUMNS):
if header.lower() != expected_header.lower():
raise Exception("Unknown header value encountered while processing collectl playback - %s" % header)
raise Exception(f"Unknown header value encountered while processing collectl playback - {header}")
# First row, check contains correct header.
current_interval = CollectlProcessInterval()
@@ -42,7 +42,7 @@ class CollectlFormatter(formatting.JobMetricFormatter):
else:
_, stat_type, resource_type = key.split("_", 2)
if resource_type.startswith("Vm"):
value_str = "%s KB" % int(value)
value_str = f"{int(value)} KB"
elif resource_type in ["RSYS", "WSYS"] and stat_type in ["count", "max", "sum"]:
value_str = "%d (# system calls)" % int(value)
else:
@@ -63,7 +63,7 @@ class CollectlPlugin(InstrumentPlugin):
self.__configure_subsystems(kwargs)
saved_logs_path = kwargs.get("saved_logs_path", "")
if "app" in kwargs:
log.debug("Found path for saved logs: %s" % saved_logs_path)
log.debug(f"Found path for saved logs: {saved_logs_path}")
saved_logs_path = kwargs["app"].config.resolve_path(saved_logs_path)
self.saved_logs_path = saved_logs_path
self.__configure_collectl_recorder_args(kwargs)
@@ -82,7 +82,7 @@ class CollectlPlugin(InstrumentPlugin):
commands = []
# Capture PID of process so we can walk its ancestors when building
# statistics for the whole job.
commands.append('''echo "$$" > '%s' ''' % self.__pid_file(job_directory))
commands.append(f'''echo "$$" > '{self.__pid_file(job_directory)}' ''')
# Run collectl in record mode to capture process and system level
# statistics according to supplied subsystems.
commands.append(self.__collectl_record_command(job_directory))
@@ -123,7 +123,7 @@ class CollectlPlugin(InstrumentPlugin):
# Run collectl in playback and generate statistics of interest
summary_statistics = self.__summarize_process_data(pid, path)
for statistic, value in summary_statistics:
properties["process_%s" % "_".join(statistic)] = value
properties[f"process_{'_'.join(statistic)}"] = value
return properties
@@ -190,10 +190,7 @@ class CollectlPlugin(InstrumentPlugin):
redirect_to = self._instrument_file_path(job_directory, "program_output")
else:
redirect_to = "/dev/null"
return "{} > {} 2>&1 &".format(
collectl_cli.build_command_line(),
redirect_to,
)
return f"{collectl_cli.build_command_line()} > {redirect_to} 2>&1 &"
def __pid_file(self, job_directory):
return self._instrument_file_path(job_directory, "pid")
+5 -5
View File
@@ -69,18 +69,18 @@ class CorePlugin(InstrumentPlugin):
def __record_galaxy_slots_command(self, job_directory):
galaxy_slots_file = self.__galaxy_slots_file(job_directory)
return '''echo "$GALAXY_SLOTS" > '%s' ''' % galaxy_slots_file
return f'''echo "$GALAXY_SLOTS" > '{galaxy_slots_file}' '''
def __record_galaxy_memory_mb_command(self, job_directory):
galaxy_memory_mb_file = self.__galaxy_memory_mb_file(job_directory)
return '''echo "$GALAXY_MEMORY_MB" > '%s' ''' % galaxy_memory_mb_file
return f'''echo "$GALAXY_MEMORY_MB" > '{galaxy_memory_mb_file}' '''
def __record_seconds_since_epoch_to_file(self, job_directory, name):
path = self._instrument_file_path(job_directory, "epoch_%s" % name)
return 'date +"%s" > ' + path
path = self._instrument_file_path(job_directory, f"epoch_{name}")
return f"date +\"%s\" > {path}"
def __read_seconds_since_epoch(self, job_directory, name):
path = self._instrument_file_path(job_directory, "epoch_%s" % name)
path = self._instrument_file_path(job_directory, f"epoch_{name}")
return self.__read_integer(path)
def __galaxy_slots_file(self, job_directory):
@@ -15,7 +15,7 @@ class CpuInfoFormatter(formatting.JobMetricFormatter):
def format(self, key, value):
if key == "processor_count":
return "Processor Count", "%s" % int(value)
return "Processor Count", f"{int(value)}"
else:
return key, value
@@ -31,7 +31,7 @@ class CpuInfoPlugin(InstrumentPlugin):
self.verbose = util.asbool(kwargs.get("verbose", False))
def pre_execute_instrument(self, job_directory):
return "cat /proc/cpuinfo > '%s'" % self.__instrument_cpuinfo_path(job_directory)
return f"cat /proc/cpuinfo > '{self.__instrument_cpuinfo_path(job_directory)}'"
def job_properties(self, job_id, job_directory):
properties = {}
+1 -1
View File
@@ -30,7 +30,7 @@ class EnvPlugin(InstrumentPlugin):
def pre_execute_instrument(self, job_directory):
""" Use env to dump all environment variables to a file.
"""
return "env > '%s'" % self.__env_file(job_directory)
return f"env > '{self.__env_file(job_directory)}'"
def post_execute_instrument(self, job_directory):
return None
@@ -23,7 +23,7 @@ class HostnamePlugin(InstrumentPlugin):
pass
def pre_execute_instrument(self, job_directory):
return "hostname -f > '%s'" % self.__instrument_hostname_path(job_directory)
return f"hostname -f > '{self.__instrument_hostname_path(job_directory)}'"
def job_properties(self, job_id, job_directory):
with open(self.__instrument_hostname_path(job_directory)) as f:
@@ -33,7 +33,7 @@ class MemInfoPlugin(InstrumentPlugin):
self.verbose = util.asbool(kwargs.get("verbose", False))
def pre_execute_instrument(self, job_directory):
return "cat /proc/meminfo > '%s'" % self.__instrument_meminfo_path(job_directory)
return f"cat /proc/meminfo > '{self.__instrument_meminfo_path(job_directory)}'"
def job_properties(self, job_id, job_directory):
properties = {}
@@ -20,7 +20,7 @@ class UnamePlugin(InstrumentPlugin):
self.uname_args = kwargs.get("args", "-a")
def pre_execute_instrument(self, job_directory):
return "uname {} > '{}'".format(self.uname_args, self.__instrument_uname_path(job_directory))
return f"uname {self.uname_args} > '{self.__instrument_uname_path(job_directory)}'"
def job_properties(self, job_id, job_directory):
properties = {}
+20 -20
View File
@@ -130,7 +130,7 @@ class JobToolConfiguration(Bunch):
def config_exception(e, file):
abs_path = os.path.abspath(file)
message = 'Problem parsing the XML in file %s, ' % abs_path
message = f'Problem parsing the XML in file {abs_path}, '
message += 'please correct the indicated portion of the file and restart Galaxy. '
message += unicodify(e)
log.exception(message)
@@ -157,7 +157,7 @@ def job_config_xml_to_dict(config, root):
kwds=runner_kwds)
runners[plugin_id] = runner_info
else:
log.error('Unknown plugin type: %s' % plugin.get('type'))
log.error(f"Unknown plugin type: {plugin.get('type')}")
for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type')):
if plugin.get('id') == 'dynamic' and plugin.get('type') == 'runner':
@@ -263,7 +263,7 @@ def job_config_xml_to_dict(config, root):
limit_dict = {}
for key in ['type', 'tag', 'id', 'window']:
if key == 'type' and key.startswith('destination_'):
key = 'environment_%s' % key[len("destination_"):]
key = f"environment_{key[len('destination_'):]}"
value = limit.get(key)
if value:
limit_dict[key] = value
@@ -498,7 +498,7 @@ class JobConfiguration(ConfiguresHandlers):
for limit_dict in job_config_dict.get("limits", []):
limit_type = limit_dict.get('type')
if limit_type.startswith("environment_"):
limit_type = 'destination_%s' % limit_type[len("environment_"):]
limit_type = f"destination_{limit_type[len('environment_'):]}"
limit_value = limit_dict.get("value")
# concurrent_jobs renamed to destination_user_concurrent_jobs in job_conf.xml
@@ -536,7 +536,7 @@ class JobConfiguration(ConfiguresHandlers):
:type tree: ``lxml.etree._Element``
"""
root = tree.getroot()
log.debug('Loading job configuration from %s' % self.app.config.job_config_file)
log.debug(f'Loading job configuration from {self.app.config.job_config_file}')
job_config_dict = job_config_xml_to_dict(self.app.config, root)
return job_config_dict
@@ -785,7 +785,7 @@ class JobConfiguration(ConfiguresHandlers):
# Name to load was specified as '<module>'
if '.' not in load:
# For legacy reasons, try from galaxy.jobs.runners first if there's no '.' in the name
module_name = 'galaxy.jobs.runners.' + load
module_name = f"galaxy.jobs.runners.{load}"
try:
module = __import__(module_name)
except ImportError:
@@ -807,14 +807,14 @@ class JobConfiguration(ConfiguresHandlers):
assert module.__all__
class_names = module.__all__
except AssertionError:
log.error('Runner "%s" does not contain a list of exported classes in __all__' % load)
log.error(f'Runner "{load}" does not contain a list of exported classes in __all__')
continue
for class_name in class_names:
runner_class = getattr(module, class_name)
try:
assert issubclass(runner_class, BaseJobRunner)
except TypeError:
log.warning("A non-class name was found in __all__, ignoring: %s" % id)
log.warning(f"A non-class name was found in __all__, ignoring: {id}")
continue
except AssertionError:
log.warning(f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}")
@@ -1007,7 +1007,7 @@ class JobWrapper(HasResourceParameters):
return self.app.config.use_tasked_jobs and self.tool.parallelism
def get_job_runner_url(self):
log.warning('(%s) Job runner URLs are deprecated, use destinations instead.' % self.job_id)
log.warning(f'({self.job_id}) Job runner URLs are deprecated, use destinations instead.')
return self.job_destination.url
def get_parallelism(self):
@@ -1079,7 +1079,7 @@ class JobWrapper(HasResourceParameters):
job = self.get_job()
for p in job.parameters:
if p.name == "__validate_outputs__":
log.info("validate... %s" % p.value)
log.info(f"validate... {p.value}")
return loads(p.value)
return False
@@ -1238,7 +1238,7 @@ class JobWrapper(HasResourceParameters):
# Restore parameters from the database
job = self.get_job()
if job.user is None and job.galaxy_session is None:
raise Exception('Job %s has no user and no session.' % job.id)
raise Exception(f'Job {job.id} has no user and no session.')
return job
def _get_tool_evaluator(self, job):
@@ -1521,10 +1521,10 @@ class JobWrapper(HasResourceParameters):
dataset.info = (dataset.info or '')
if context['stdout'].strip():
# Ensure white space between entries
dataset.info = dataset.info.rstrip() + "\n" + context['stdout'].strip()
dataset.info = f"{dataset.info.rstrip()}\n{context['stdout'].strip()}"
if context['stderr'].strip():
# Ensure white space between entries
dataset.info = dataset.info.rstrip() + "\n" + context['stderr'].strip()
dataset.info = f"{dataset.info.rstrip()}\n{context['stderr'].strip()}"
dataset.tool_version = self.version_string
dataset.set_size()
if 'uuid' in context:
@@ -1672,7 +1672,7 @@ class JobWrapper(HasResourceParameters):
else:
# Prior to fail we need to set job.state
job.set_state(final_job_state)
return self.fail("Job %s's output dataset(s) could not be read" % job.id)
return self.fail(f"Job {job.id}'s output dataset(s) could not be read")
job_context = ExpressionContext(dict(stdout=job.stdout, stderr=job.stderr))
if extended_metadata:
@@ -1860,7 +1860,7 @@ class JobWrapper(HasResourceParameters):
job_metrics_directory = job_metrics_directory or self.working_directory
per_plugin_properties = self.app.job_metrics.collect_properties(job.destination_id, self.job_id, job_metrics_directory)
if per_plugin_properties:
log.info("Collecting metrics for {} {} in {}".format(type(has_metrics).__name__, getattr(has_metrics, 'id', None), job_metrics_directory))
log.info(f"Collecting metrics for {type(has_metrics).__name__} {getattr(has_metrics, 'id', None)} in {job_metrics_directory}")
for plugin, properties in per_plugin_properties.items():
for metric_name, metric_value in properties.items():
if metric_value is not None:
@@ -2046,7 +2046,7 @@ class JobWrapper(HasResourceParameters):
elif target == "pwd":
return os.path.join(working_directory, "working")
else:
raise Exception("Unknown target type [%s]" % target)
raise Exception(f"Unknown target type [{target}]")
def get_tool_provided_job_metadata(self):
if self.tool_provided_job_metadata is not None:
@@ -2146,7 +2146,7 @@ class JobWrapper(HasResourceParameters):
except ValueError:
# File exists, but is not fully populated yet
return False
log.debug("found container runtime %s" % container_runtime)
log.debug(f"found container runtime {container_runtime}")
self.app.interactivetool_manager.configure_entry_points(job, container_runtime)
return True
container_exception_path = os.path.join(working_directory, "container_monitor_exception.txt")
@@ -2192,7 +2192,7 @@ class JobWrapper(HasResourceParameters):
with open(container_config, "w") as f:
json.dump(container_config_dict, f)
return "(python '%s'/lib/galaxy_ext/container_monitor/monitor.py &); sleep 1 " % exec_dir
return f"(python '{exec_dir}'/lib/galaxy_ext/container_monitor/monitor.py &); sleep 1 "
@property
def user(self):
@@ -2204,7 +2204,7 @@ class JobWrapper(HasResourceParameters):
elif job.history is not None and job.history.user is not None:
return job.history.user.email
elif job.galaxy_session is not None:
return 'anonymous@' + job.galaxy_session.remote_addr.split()[-1]
return f"anonymous@{job.galaxy_session.remote_addr.split()[-1]}"
else:
return 'anonymous@unknown'
@@ -2374,7 +2374,7 @@ class TaskWrapper(JobWrapper):
return self.extra_filenames
def fail(self, message, exception=False):
log.error("TaskWrapper Failure %s" % message)
log.error(f"TaskWrapper Failure {message}")
self.status = 'error'
# How do we want to handle task failure? Fail the job and let it clean up?
+11 -11
View File
@@ -30,9 +30,9 @@ class DefaultJobAction:
@classmethod
def get_short_str(cls, pja):
if pja.action_arguments:
return "{} -> {}".format(pja.action_type, escape(pja.action_arguments))
return f"{pja.action_type} -> {escape(pja.action_arguments)}"
else:
return "%s" % pja.action_type
return f"{pja.action_type}"
class EmailAction(DefaultJobAction):
@@ -47,17 +47,17 @@ class EmailAction(DefaultJobAction):
try:
frm = app.config.email_from
history_id_encoded = app.security.encode_id(job.history_id)
link = app.config.galaxy_infrastructure_url + "/histories/view?id=" + history_id_encoded
link = f"{app.config.galaxy_infrastructure_url}/histories/view?id={history_id_encoded}"
if frm is None:
if action.action_arguments and 'host' in action.action_arguments:
host = action.action_arguments['host']
else:
host = socket.getfqdn()
frm = 'galaxy-no-reply@%s' % host
frm = f'galaxy-no-reply@{host}'
to = job.user.email
subject = "Galaxy job completion notification from history '%s'" % (job.history.name)
subject = f"Galaxy job completion notification from history '{job.history.name}'"
outdata = ',\n'.join(ds.dataset.display_name() for ds in job.output_datasets)
body = "Your Galaxy job generating dataset(s):\n\n{}\n\nis complete as of {}. Click the link below to access your data: \n{}".format(outdata, datetime.datetime.now().strftime("%I:%M"), link)
body = f"Your Galaxy job generating dataset(s):\n\n{outdata}\n\nis complete as of {datetime.datetime.now().strftime('%I:%M')}. Click the link below to access your data: \n{link}"
send_mail(frm, to, subject, body, app.config)
except Exception as e:
log.error("EmailAction PJA Failed, exception: %s", unicodify(e))
@@ -65,7 +65,7 @@ class EmailAction(DefaultJobAction):
@classmethod
def get_short_str(cls, pja):
if pja.action_arguments and 'host' in pja.action_arguments:
return "Email the current user from server %s when this job is complete." % escape(pja.action_arguments['host'])
return f"Email the current user from server {escape(pja.action_arguments['host'])} when this job is complete."
else:
return "Email the current user when this job is complete."
@@ -191,7 +191,7 @@ class RenameDatasetAction(DefaultJobAction):
if len(fields) > 1:
temp = ""
for i in range(1, len(fields) - 1):
temp += "." + fields[i]
temp += f".{fields[i]}"
replacement += temp
elif operation == "upper":
replacement = replacement.upper()
@@ -264,7 +264,7 @@ class HideDatasetAction(DefaultJobAction):
@classmethod
def get_short_str(cls, pja):
return "Hide output '%s'." % escape(pja.output_name)
return f"Hide output '{escape(pja.output_name)}'."
class DeleteDatasetAction(DefaultJobAction):
@@ -313,7 +313,7 @@ class ColumnSetAction(DefaultJobAction):
@classmethod
def get_short_str(cls, pja):
return "Set the following metadata values:<br/>" + "<br/>".join('{} : {}'.format(escape(k), escape(v)) for k, v in pja.action_arguments.items())
return f"Set the following metadata values:<br/>{'<br/>'.join('{} : {}'.format(escape(k), escape(v)) for k, v in pja.action_arguments.items())}"
class SetMetadataAction(DefaultJobAction):
@@ -440,7 +440,7 @@ class TagDatasetAction(DefaultJobAction):
cls.direction,
escape(pja.output_name))
else:
return "%s Tag action used without a tag specified. No tag will be added." % cls.action
return f"{cls.action} Tag action used without a tag specified. No tag will be added."
class RemoveTagDatasetAction(TagDatasetAction):
+5 -10
View File
@@ -124,7 +124,7 @@ def build_command(
if include_metadata and job_wrapper.requires_setting_metadata:
working_directory = remote_job_directory or job_wrapper.working_directory
commands_builder.append_command("cd '%s'" % working_directory)
commands_builder.append_command(f"cd '{working_directory}'")
__handle_metadata(commands_builder, job_wrapper, runner, remote_command_params)
return commands_builder.build()
@@ -167,7 +167,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_
# https://github.com/galaxyproject/galaxy/pull/8449
for_pulsar = False
if 'script_directory' in remote_command_params:
commands = "{} {}".format(shell, join(remote_command_params['script_directory'], script_name))
commands = f"{shell} {join(remote_command_params['script_directory'], script_name)}"
for_pulsar = True
if not for_pulsar:
commands += " > ../outputs/tool_stdout 2> ../outputs/tool_stderr"
@@ -263,9 +263,7 @@ class CommandsBuilder:
def prepend_command(self, command, sep=";"):
if command:
self.commands = "{}{} {}".format(command,
sep,
self.commands)
self.commands = f"{command}{sep} {self.commands}"
return self
def prepend_commands(self, commands):
@@ -273,9 +271,7 @@ class CommandsBuilder:
def append_command(self, command, sep=';'):
if command:
self.commands = "{}{} {}".format(self.commands,
sep,
command)
self.commands = f"{self.commands}{sep} {command}"
return self
def append_commands(self, commands):
@@ -288,8 +284,7 @@ trap 'rm "$out" "$err"' EXIT
tee -a stdout.log < "$out" &
tee -a stderr.log < "$err" >&2 &""",
sep="")
self.append_command("> '{stdout_file}' 2> '{stderr_file}'".format(stdout_file=stdout_file,
stderr_file=stderr_file),
self.append_command(f"> '{stdout_file}' 2> '{stderr_file}'",
sep="")
def capture_return_code(self):
+71 -71
View File
@@ -352,9 +352,9 @@ class RuleValidator:
if "nice_value" in rule:
if rule["nice_value"] < -20 or rule["nice_value"] > 20:
error = "nice_value goes from -20 to 20; rule " + str(counter)
error += " in '" + str(tool) + "' has a nice_value of '"
error += str(rule["nice_value"]) + "'."
error = f"nice_value goes from -20 to 20; rule {str(counter)}"
error += f" in '{str(tool)}' has a nice_value of '"
error += f"{str(rule['nice_value'])}'."
if not return_bool:
error += " Setting nice_value to 0."
rule["nice_value"] = 0
@@ -364,8 +364,8 @@ class RuleValidator:
valid_rule = False
else:
error = "No nice_value found for rule " + str(counter) + " in '"
error += str(tool) + "'."
error = f"No nice_value found for rule {str(counter)} in '"
error += f"{str(tool)}'."
if not return_bool:
error += " Setting nice_value to 0."
rule["nice_value"] = 0
@@ -394,8 +394,8 @@ class RuleValidator:
if "fail_message" in rule:
if "destination" not in rule or rule['destination'] != "fail":
error = "Found a fail_message for rule " + str(counter)
error += " in '" + str(tool) + "', but destination is not 'fail'!"
error = f"Found a fail_message for rule {str(counter)}"
error += f" in '{str(tool)}', but destination is not 'fail'!"
if not return_bool:
error += " Setting destination to 'fail'."
if verbose:
@@ -409,12 +409,12 @@ class RuleValidator:
suggestion = None
if isinstance(rule["destination"], str):
if rule["destination"] == "fail" and "fail_message" not in rule:
error = "Missing a fail_message for rule " + str(counter)
error += " in '" + str(tool) + "'."
error = f"Missing a fail_message for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Adding generic fail_message."
message = "Invalid parameters for rule " + str(counter)
message += " in '" + str(tool) + "'."
message = f"Invalid parameters for rule {str(counter)}"
message += f" in '{str(tool)}'."
rule["fail_message"] = message
if verbose:
log.debug(error)
@@ -432,12 +432,12 @@ class RuleValidator:
for priority in rule["destination"]["priority"]:
if priority not in priority_list:
error = "Invalid priority '"
error += str(priority) + "' for rule "
error += str(counter) + " in '" + str(tool) + "'."
error += f"{str(priority)}' for rule "
error += f"{str(counter)} in '{str(tool)}'."
suggestion = get_typo_correction(priority,
priority_list, max_edit_dist)
if suggestion:
error += " Did you mean '" + str(suggestion) + "'?"
error += f" Did you mean '{str(suggestion)}'?"
if not return_bool:
error += " Ignoring..."
if verbose:
@@ -447,8 +447,8 @@ class RuleValidator:
elif not isinstance(rule["destination"]["priority"][priority], str):
error = "Cannot parse tool destination '"
error += str(rule["destination"]["priority"][priority])
error += "' for rule " + str(counter)
error += " in '" + str(tool) + "'."
error += f"' for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Ignoring..."
if verbose:
@@ -463,24 +463,24 @@ class RuleValidator:
if not is_valid:
valid_rule = False
else:
error = "No destination specified for rule " + str(counter)
error += " in '" + str(tool) + "'."
error = f"No destination specified for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Ignoring..."
if verbose:
log.debug(error)
valid_rule = False
else:
error = "No destination specified for rule " + str(counter)
error += " in '" + str(tool) + "'."
error = f"No destination specified for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Ignoring..."
if verbose:
log.debug(error)
valid_rule = False
else:
error = "No destination specified for rule " + str(counter)
error += " in '" + str(tool) + "'."
error = f"No destination specified for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Ignoring..."
if verbose:
@@ -542,8 +542,8 @@ class RuleValidator:
if upper_bound != -1 and lower_bound > upper_bound:
error = "lower_bound exceeds upper_bound for rule " + str(counter)
error += " in '" + str(tool) + "'."
error = f"lower_bound exceeds upper_bound for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Reversing bounds."
temp_upper_bound = rule["upper_bound"]
@@ -555,8 +555,8 @@ class RuleValidator:
valid_rule = False
else:
error = "Missing bounds for rule " + str(counter)
error += " in '" + str(tool) + "'."
error = f"Missing bounds for rule {str(counter)}"
error += f" in '{str(tool)}'."
if not return_bool:
error += " Ignoring rule."
rule = None
@@ -594,8 +594,8 @@ class RuleValidator:
"""
if "arguments" not in rule or not isinstance(rule["arguments"], dict):
error = "No arguments found for rule " + str(counter) + " in '"
error += str(tool) + "' despite being of type arguments."
error = f"No arguments found for rule {str(counter)} in '"
error += f"{str(tool)}' despite being of type arguments."
if not return_bool:
error += " Ignoring rule."
rule = None
@@ -638,9 +638,9 @@ class RuleValidator:
if isinstance(rule["users"], list):
for user in reversed(rule["users"]):
if not isinstance(user, str):
error = "Entry '" + str(user) + "' in users for rule "
error += str(counter) + " in tool '" + str(tool)
error += "' is in an " + "invalid format!"
error = f"Entry '{str(user)}' in users for rule "
error += f"{str(counter)} in tool '{str(tool)}"
error += "' is in an invalid format!"
if not return_bool:
error += " Ignoring entry."
if verbose:
@@ -650,9 +650,9 @@ class RuleValidator:
else:
if re.match(emailregex, user) is None:
error = "Supplied email '" + str(user)
error += "' for rule " + str(counter) + " in tool '"
error += str(tool) + "' is in " + "an invalid format!"
error = f"Supplied email '{str(user)}"
error += f"' for rule {str(counter)} in tool '"
error += f"{str(tool)}' is in an invalid format!"
if not return_bool:
error += " Ignoring email."
if verbose:
@@ -672,8 +672,8 @@ class RuleValidator:
# post-processing checking to make sure we didn't just remove all the users
# if we did, we should ignore the rule
if rule is not None and rule["users"] is not None and len(rule["users"]) == 0:
error = "No valid user emails were specified for rule " + str(counter)
error += " in tool '" + str(tool) + "'!"
error = f"No valid user emails were specified for rule {str(counter)}"
error += f" in tool '{str(tool)}'!"
if not return_bool:
error += " Ignoring rule."
rule = None
@@ -786,7 +786,7 @@ def validate_destination(app, destination: str, err_message: str, err_message_co
if not valid_destination:
error = err_message % err_message_contents
if suggestion:
error += " Did you mean '" + suggestion + "'?"
error += f" Did you mean '{suggestion}'?"
if not return_bool:
error += " Ignoring..."
if verbose:
@@ -828,7 +828,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
else:
valid_config = False
if obj:
log.debug("Verbose value '" + str(obj['verbose']) + "' is not True or False! Falling back to verbose...")
log.debug(f"Verbose value '{str(obj['verbose'])}' is not True or False! Falling back to verbose...")
verbose = True
if not return_bool and verbose:
@@ -891,7 +891,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
suggestion = get_typo_correction(obj['default_priority'],
priority_list, max_edit_dist)
if suggestion:
error += " Did you mean '" + str(suggestion) + "'?"
error += f" Did you mean '{str(suggestion)}'?"
if verbose:
log.debug(error)
else:
@@ -946,17 +946,17 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
suggestion = get_typo_correction(curr['priority'],
priority_list, max_edit_dist)
if suggestion:
error += " Did you mean '" + str(suggestion) + "'?"
error += f" Did you mean '{str(suggestion)}'?"
if verbose:
log.debug(error)
valid_config = False
else:
error = "User '" + user + "' is missing a priority!"
error = f"User '{user}' is missing a priority!"
if verbose:
log.debug(error)
valid_config = False
else:
error = "User '" + user + "' is missing a priority!"
error = f"User '{user}' is missing a priority!"
if verbose:
log.debug(error)
valid_config = False
@@ -1027,13 +1027,13 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
suggestion = get_typo_correction(priority,
priority_list, max_edit_dist)
if suggestion:
error += " Did you mean '" + str(suggestion) + "'?"
error += f" Did you mean '{str(suggestion)}'?"
if verbose:
log.debug(error)
valid_config = False
else:
error = "No default priority destinations specified"
error += " for " + str(tool) + " in config!"
error += f" for {str(tool)} in config!"
if verbose:
log.debug(error)
valid_config = False
@@ -1081,8 +1081,8 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
# if rule['rule_type'] in available_rule_types
else:
error = "Unrecognized rule_type '"
error += rule['rule_type'] + "' "
error += "found in '" + str(tool) + "'. "
error += f"{rule['rule_type']}' "
error += f"found in '{str(tool)}'. "
if not return_bool:
error += "Ignoring..."
if verbose:
@@ -1094,7 +1094,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
counter += 1
error = "No rule_type found for rule "
error += str(counter)
error += " in '" + str(tool) + "'."
error += f" in '{str(tool)}'."
if verbose:
log.debug(error)
valid_config = False
@@ -1102,7 +1102,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
# if "rules" in curr and isinstance(curr['rules'], list):
elif not tool_has_default:
valid_config = False
error = "Tool '" + str(tool) + "' does not have"
error = f"Tool '{str(tool)}' does not have"
error += " rules nor a default_destination!"
if verbose:
log.debug(error)
@@ -1110,7 +1110,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
# if obj['tools'][tool] is not None:
else:
valid_config = False
error = "Config section for tool '" + str(tool) + "' is blank!"
error = f"Config section for tool '{str(tool)}' is blank!"
if verbose:
log.debug(error)
@@ -1128,7 +1128,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False):
# quickly run through categories to detect unrecognized types
for category in obj.keys():
if category not in valid_categories:
error = "Unrecognized category '" + category
error = f"Unrecognized category '{category}"
error += "' found in config file!"
if verbose:
log.debug(error)
@@ -1187,9 +1187,9 @@ def bytes_to_str(size, unit="YB"):
i = 0
try:
return_str = "{:.2f} {}".format(size_changer, units[i])
return_str = f"{size_changer:.2f} {units[i]}"
except (ValueError, TypeError):
return_str = "%s" % (size_changer)
return_str = f"{size_changer}"
return return_str
@@ -1225,7 +1225,7 @@ def str_to_bytes(size):
try:
curr_size = float(curr_size)
except ValueError:
error = "Unable to convert size " + str(size)
error = f"Unable to convert size {str(size)}"
raise MalformedYMLException(error)
# Get the unit and convert to bytes
@@ -1234,7 +1234,7 @@ def str_to_bytes(size):
for _ in range(pos, 1, -1):
curr_size *= 1024
except ValueError:
error = "Unable to convert size " + str(size)
error = f"Unable to convert size {str(size)}"
raise MalformedYMLException(error)
except NameError:
pass
@@ -1343,7 +1343,7 @@ def map_tool_to_destination(
if inp_data[da] is not None and os.path.isfile(inp_data[da].file_name):
num_input_datasets += 1
if verbose:
message = "Loading file: " + str(da)
message = f"Loading file: {str(da)}"
message += str(inp_data[da].file_name)
log.debug(message)
@@ -1367,15 +1367,15 @@ def map_tool_to_destination(
except AttributeError:
# Otherwise, say that input isn't a file
if verbose:
log.debug("Not a file: " + str(inp_data[da]))
log.debug(f"Not a file: {str(inp_data[da])}")
if verbose:
if filesize_rule_present:
log.debug("Total size: " + bytes_to_str(file_size))
log.debug(f"Total size: {bytes_to_str(file_size)}")
if records_rule_present:
log.debug("Total amount of records: " + str(records))
log.debug(f"Total amount of records: {str(records)}")
if num_input_datasets_rule_present:
log.debug("Total number of files: " + str(num_input_datasets))
log.debug(f"Total number of files: {str(num_input_datasets)}")
matched_rule = None
user_authorized = None
@@ -1516,7 +1516,7 @@ def map_tool_to_destination(
except KeyError:
matched = False
if verbose:
error = "Argument '" + str(arg)
error = f"Argument '{str(arg)}"
error += "' not recognized!"
log.debug(error)
@@ -1528,15 +1528,15 @@ def map_tool_to_destination(
# if user_authorized
else:
if verbose:
error = "User email '" + str(user_email) + "' not "
error = f"User email '{str(user_email)}' not "
error += "specified in list of authorized users for "
error += "rule " + str(rule_counter) + " in tool '"
error += str(tool.old_id) + "'! Ignoring rule."
error += f"rule {str(rule_counter)} in tool '"
error += f"{str(tool.old_id)}'! Ignoring rule."
log.debug(error)
# if str(tool.old_id) in config
else:
error = "Tool '" + str(tool.old_id) + "' not specified in config. "
error = f"Tool '{str(tool.old_id)}' not specified in config. "
error += "Using default destination."
if verbose:
log.debug(error)
@@ -1565,7 +1565,7 @@ def map_tool_to_destination(
# if "default_destination" in config
else:
destination = "fail"
fail_message = "Job '" + str(tool.old_id) + "' failed; "
fail_message = f"Job '{str(tool.old_id)}' failed; "
fail_message += "no global default destination specified in config!"
# if fail_message is not None
@@ -1582,11 +1582,11 @@ def map_tool_to_destination(
if config is not None:
if destination == "fail":
output = "An error occurred: " + fail_message
output = f"An error occurred: {fail_message}"
log.debug(output)
else:
output = "Running '" + str(tool.old_id) + "' with '"
output += destination + "'."
output = f"Running '{str(tool.old_id)}' with '"
output += f"{destination}'."
log.debug(output)
return destination
@@ -1632,7 +1632,7 @@ def get_destination_list_from_job_config(job_config_location) -> set:
possible_job_conf_path = os.path.join(config_location, f)
if os.path.isfile(possible_job_conf_path):
job_config_location = possible_job_conf_path
message += "using '%s'. *" % f
message += f"using '{f}'. *"
break
else:
message += ("and no default job configs in 'config/'. "
@@ -1650,7 +1650,7 @@ def get_destination_list_from_job_config(job_config_location) -> set:
destination_list.add(destination.get("id"))
else:
error = "Destination ID '" + str(destination)
error = f"Destination ID '{str(destination)}"
error += "' in job configuration file cannot be"
error += " parsed. Things may not work as expected!"
log.debug(error)
@@ -1783,7 +1783,7 @@ if __name__ == '__main__':
'-j', '--job-config', dest='job_config')
parser.add_argument(
'-V', '--version', action='version', version="%(prog)s " + __version__)
'-V', '--version', action='version', version=f"%(prog)s {__version__}")
args = parser.parse_args()
+14 -14
View File
@@ -247,7 +247,7 @@ class JobHandlerQueue(Monitors):
self.job_wrapper(job).fail('This tool was disabled before the job completed. Please contact your Galaxy administrator.')
elif job.job_runner_name is not None and job.job_runner_external_id is None:
# This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner.
log.debug("(%s) Job runner assigned but no external ID recorded, adding to the job handler queue" % job.id)
log.debug(f"({job.id}) Job runner assigned but no external ID recorded, adding to the job handler queue")
job.job_runner_name = None
if self.track_jobs_in_database:
job.set_state(model.Job.states.NEW)
@@ -261,7 +261,7 @@ class JobHandlerQueue(Monitors):
job_destination.id = 'legacy_url'
job_wrapper.set_job_destination(job_destination, job.job_runner_external_id)
self.dispatcher.recover(job, job_wrapper)
log.info('(%s) Converted job from a URL to a destination and recovered' % (job.id))
log.info(f'({job.id}) Converted job from a URL to a destination and recovered')
elif job.job_runner_name is None:
# Never (fully) dispatched
log.debug(f"({job.id}) No job runner assigned and job still in '{job.state}' state, adding to the job handler queue")
@@ -512,20 +512,20 @@ class JobHandlerQueue(Monitors):
if hda_deleted or dataset_deleted:
if dataset_purged:
# If the dataset has been purged we can't resume the job by undeleting the input
jobs_to_fail[job_id].append("Input dataset '%s' was deleted before the job started" % hda_name)
jobs_to_fail[job_id].append(f"Input dataset '{hda_name}' was deleted before the job started")
else:
jobs_to_pause[job_id].append("Input dataset '%s' was deleted before the job started" % hda_name)
jobs_to_pause[job_id].append(f"Input dataset '{hda_name}' was deleted before the job started")
elif hda_state == model.HistoryDatasetAssociation.states.FAILED_METADATA:
jobs_to_pause[job_id].append("Input dataset '%s' failed to properly set metadata" % hda_name)
jobs_to_pause[job_id].append(f"Input dataset '{hda_name}' failed to properly set metadata")
elif dataset_state == model.Dataset.states.PAUSED:
jobs_to_pause[job_id].append("Input dataset '%s' was paused before the job started" % hda_name)
jobs_to_pause[job_id].append(f"Input dataset '{hda_name}' was paused before the job started")
elif dataset_state == model.Dataset.states.ERROR:
jobs_to_pause[job_id].append("Input dataset '%s' is in error state" % hda_name)
jobs_to_pause[job_id].append(f"Input dataset '{hda_name}' is in error state")
elif dataset_state != model.Dataset.states.OK:
jobs_to_ignore[job_id].append(f"Input dataset '{hda_name}' is in {dataset_state} state")
for job_id in sorted(jobs_to_pause):
pause_message = ", ".join(jobs_to_pause[job_id])
pause_message = "%s. To resume this job fix the input dataset(s)." % pause_message
pause_message = f"{pause_message}. To resume this job fix the input dataset(s)."
job, job_wrapper = self.job_pair_for_id(job_id)
try:
job_wrapper.pause(job=job, message=pause_message)
@@ -604,7 +604,7 @@ class JobHandlerQueue(Monitors):
if failure_message == DEFAULT_JOB_PUT_FAILURE_MESSAGE:
log.exception('Failed to generate job destination')
else:
log.debug("Intentionally failing job with message (%s)" % failure_message)
log.debug(f"Intentionally failing job with message ({failure_message})")
job_wrapper.fail(failure_message)
return JOB_ERROR, job_destination
# job is ready to run, check limits
@@ -667,10 +667,10 @@ class JobHandlerQueue(Monitors):
return JOB_INPUT_DELETED
# an error in the input data causes us to bail immediately
elif idata.state == idata.states.ERROR:
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail("input data %s is in error state" % (idata.hid))
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} is in error state")
return JOB_INPUT_ERROR
elif idata.state == idata.states.FAILED_METADATA:
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail("input data %s failed to properly set metadata" % (idata.hid))
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} failed to properly set metadata")
return JOB_INPUT_ERROR
elif idata.state != idata.states.OK and not (idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id):
# need to requeue
@@ -806,7 +806,7 @@ class JobHandlerQueue(Monitors):
if count >= self.app.job_config.limits.anonymous_user_concurrent_jobs:
return JOB_WAIT
else:
log.warning('Job %s is not associated with a user or session so job concurrency limit cannot be checked.' % job.id)
log.warning(f'Job {job.id} is not associated with a user or session so job concurrency limit cannot be checked.')
return JOB_READY
def __cache_total_job_count_per_destination(self):
@@ -1022,7 +1022,7 @@ class DefaultJobDispatcher:
# URLs can have their URL params converted to the destination's param
# dict by the plugin.
self.app.job_config.convert_legacy_destinations(self.job_runners)
log.debug("Loaded job runners plugins: " + ':'.join(self.job_runners.keys()))
log.debug(f"Loaded job runners plugins: {':'.join(self.job_runners.keys())}")
def __get_runner_name(self, job_wrapper):
if job_wrapper.can_split():
@@ -1097,4 +1097,4 @@ class DefaultJobDispatcher:
failures.append(name)
log.exception("Failed to shutdown runner %s", name)
if failures:
raise Exception("Failed to shutdown runners: %s" % ', '.join(failures))
raise Exception(f"Failed to shutdown runners: {', '.join(failures)}")
+1 -1
View File
@@ -73,7 +73,7 @@ class JobManager:
tool_id = tool.id
configured_handler = tool.get_configured_job_handler(job.params)
if configured_handler is not None:
p = " (with job params: %s)" % str(job.params) if job.params else ""
p = f" (with job params: {str(job.params)})" if job.params else ""
log.debug("(%s) Configured job handler for tool '%s'%s is: %s", job.log_str(), tool_id, p, configured_handler)
queue_callback = partial(self._queue_callback, job, tool_id)
message_callback = partial(self._message_callback, job)
+2 -2
View File
@@ -197,7 +197,7 @@ class JobRunnerMapper:
elif expand_type in STOCK_RULES:
expand_function = STOCK_RULES[expand_type]
else:
raise JobMappingConfigurationException("Unhandled dynamic job runner type specified - %s" % expand_type)
raise JobMappingConfigurationException(f"Unhandled dynamic job runner type specified - {expand_type}")
return self.__handle_rule(expand_function, destination)
@@ -214,7 +214,7 @@ class JobRunnerMapper:
def __determine_job_destination(self, params, raw_job_destination=None):
if self.job_wrapper.tool is None:
raise JobMappingException(
"Can't map job to destination, tool '%s' is unavailable" % self.job_wrapper.get_job().tool_id
f"Can't map job to destination, tool '{self.job_wrapper.get_job().tool_id}' is unavailable"
)
if raw_job_destination is None:
raw_job_destination = self.job_wrapper.tool.get_job_destination(params)
+13 -13
View File
@@ -130,7 +130,7 @@ class BaseJobRunner:
try:
action_str = f'galaxy.jobs.runners.{self.__class__.__name__.lower()}.{name}'
action_timer = self.app.execution_timer_factory.get_timer(
'internals.%s' % action_str,
f'internals.{action_str}',
'job runner action %s for job ${job_id} executed' % (action_str)
)
method(arg)
@@ -540,7 +540,7 @@ class BaseJobRunner:
job_wrapper.finish(tool_stdout, tool_stderr, exit_code, check_output_detected_state=check_output_detected_state, job_stdout=job_stdout, job_stderr=job_stderr)
except Exception:
log.exception("({}/{}) Job wrapper finish method failed".format(job_id or '', external_job_id or ''))
log.exception(f"({job_id or ''}/{external_job_id or ''}) Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
@@ -566,19 +566,19 @@ class JobState:
id_tag = self.job_wrapper.get_id_tag()
if files_dir is not None:
self.job_file = JobState.default_job_file(files_dir, id_tag)
self.output_file = os.path.join(files_dir, 'galaxy_%s.o' % id_tag)
self.error_file = os.path.join(files_dir, 'galaxy_%s.e' % id_tag)
self.output_file = os.path.join(files_dir, f'galaxy_{id_tag}.o')
self.error_file = os.path.join(files_dir, f'galaxy_{id_tag}.e')
self.exit_code_file = default_exit_code_file(files_dir, id_tag)
job_name = 'g%s' % id_tag
job_name = f'g{id_tag}'
if self.job_wrapper.tool.old_id:
job_name += '_%s' % self.job_wrapper.tool.old_id
job_name += f'_{self.job_wrapper.tool.old_id}'
if not self.redact_email_in_job_name and self.job_wrapper.user:
job_name += '_%s' % self.job_wrapper.user
self.job_name = ''.join(x if x in (string.ascii_letters + string.digits + '_') else '_' for x in job_name)
job_name += f'_{self.job_wrapper.user}'
self.job_name = ''.join(x if x in (f"{string.ascii_letters + string.digits}_") else '_' for x in job_name)
@staticmethod
def default_job_file(files_dir, id_tag):
return os.path.join(files_dir, 'galaxy_%s.sh' % id_tag)
return os.path.join(files_dir, f'galaxy_{id_tag}.sh')
def read_exit_code(self):
return read_exit_code_from(self.exit_code_file, self.job_wrapper.get_id_tag())
@@ -591,10 +591,10 @@ class JobState:
# TODO: Move this prefix stuff to a method so we don't have dispatch on attributes we may or may
# not have.
if not hasattr(self, "job_id"):
prefix = "(%s)" % self.job_wrapper.get_id_tag()
prefix = f"({self.job_wrapper.get_id_tag()})"
else:
prefix = f"({self.job_wrapper.get_id_tag()}/{self.job_id})"
log.debug("{} Unable to cleanup {}: {}".format(prefix, file, unicodify(e)))
log.debug(f"{prefix} Unable to cleanup {file}: {unicodify(e)}")
class AsynchronousJobState(JobState):
@@ -672,7 +672,7 @@ class AsynchronousJobRunner(BaseJobRunner, Monitors):
self.monitor_queue = Queue()
def _init_monitor_thread(self):
name = "%s.monitor_thread" % self.runner_name
name = f"{self.runner_name}.monitor_thread"
super()._init_monitor_thread(name=name, target=self.monitor, start=True, config=self.app.config)
def handle_stop(self):
@@ -709,7 +709,7 @@ class AsynchronousJobRunner(BaseJobRunner, Monitors):
def shutdown(self):
"""Attempts to gracefully shut down the monitor thread"""
log.info("%s: Sending stop signal to monitor thread" % self.runner_name)
log.info(f"{self.runner_name}: Sending stop signal to monitor thread")
self.monitor_queue.put(STOP_SIGNAL)
# Call the parent's shutdown method to stop workers
self.shutdown_monitor()
+4 -4
View File
@@ -147,10 +147,10 @@ class ChronosJobRunner(AsynchronousJobRunner):
@handle_exception_call
def queue_job(self, job_wrapper):
LOGGER.debug("Starting queue_job for job " + job_wrapper.get_id_tag())
LOGGER.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
if not self.prepare_job(job_wrapper, include_metadata=False,
modify_command_for_container=False):
LOGGER.debug("Not ready " + job_wrapper.get_id_tag())
LOGGER.debug(f"Not ready {job_wrapper.get_id_tag()}")
return
job_destination = job_wrapper.job_destination
chronos_job_spec = self._get_job_spec(job_wrapper)
@@ -279,7 +279,7 @@ class ChronosJobRunner(AsynchronousJobRunner):
if not os.path.exists(job_wrapper.working_directory):
LOGGER.error("No working directory found")
path = job_wrapper.working_directory + '/chronos_' + job_wrapper.get_id_tag() + '.sh'
path = f"{job_wrapper.working_directory}/chronos_{job_wrapper.get_id_tag()}.sh"
mode = 0o755
with open(path, 'w', encoding='utf-8') as f:
@@ -295,7 +295,7 @@ class ChronosJobRunner(AsynchronousJobRunner):
template = {
'async': False,
# 'command': job_wrapper.runner_command_line,
'command': '$SHELL ' + command_script_path,
'command': f"$SHELL {command_script_path}",
'owner': self.runner_params['owner'],
'disabled': False,
'schedule': 'R1//PT1S',
+4 -4
View File
@@ -43,8 +43,8 @@ class ShellJobRunner(AsynchronousJobRunner):
params = {}
shell_params, job_params = url.split('/')[2:4]
# split 'foo=bar&baz=quux' into { 'foo' : 'bar', 'baz' : 'quux' }
shell_params = {'shell_' + k: v for k, v in [kv.split('=', 1) for kv in shell_params.split('&')]}
job_params = {'job_' + k: v for k, v in [kv.split('=', 1) for kv in job_params.split('&')]}
shell_params = {f"shell_{k}": v for k, v in [kv.split('=', 1) for kv in shell_params.split('&')]}
job_params = {f"job_{k}": v for k, v in [kv.split('=', 1) for kv in job_params.split('&')]}
params.update(shell_params)
params.update(job_params)
log.debug(f"Converted URL '{url}' to destination runner=cli, params={params}")
@@ -83,7 +83,7 @@ class ShellJobRunner(AsynchronousJobRunner):
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
log.exception("(%s) failure writing job script" % galaxy_id_tag)
log.exception(f"({galaxy_id_tag}) failure writing job script")
job_wrapper.fail("failure preparing job script", exception=True)
return
@@ -104,7 +104,7 @@ class ShellJobRunner(AsynchronousJobRunner):
# Strip and split to get job ID.
external_job_id = stdout.strip().split()[-1]
if not external_job_id:
log.error('(%s) submission did not return a job identifier, failing job' % galaxy_id_tag)
log.error(f'({galaxy_id_tag}) submission did not return a job identifier, failing job')
job_wrapper.fail("failure submitting job")
return
+6 -6
View File
@@ -83,7 +83,7 @@ class CondorJobRunner(AsynchronousJobRunner):
galaxy_slots = query_params.get('request_cpus', None)
if galaxy_slots:
galaxy_slots_statement = 'GALAXY_SLOTS="%s"; export GALAXY_SLOTS; GALAXY_SLOTS_CONFIGURED="1"; export GALAXY_SLOTS_CONFIGURED;' % galaxy_slots
galaxy_slots_statement = f'GALAXY_SLOTS="{galaxy_slots}"; export GALAXY_SLOTS; GALAXY_SLOTS_CONFIGURED="1"; export GALAXY_SLOTS_CONFIGURED;'
else:
galaxy_slots_statement = 'GALAXY_SLOTS="1"; export GALAXY_SLOTS;'
@@ -93,9 +93,9 @@ class CondorJobRunner(AsynchronousJobRunner):
job_wrapper=job_wrapper
)
cjs.user_log = os.path.join(job_wrapper.working_directory, 'galaxy_%s.condor.log' % galaxy_id_tag)
cjs.user_log = os.path.join(job_wrapper.working_directory, f'galaxy_{galaxy_id_tag}.condor.log')
cjs.register_cleanup_file_attribute('user_log')
submit_file = os.path.join(job_wrapper.working_directory, 'galaxy_%s.condor.desc' % galaxy_id_tag)
submit_file = os.path.join(job_wrapper.working_directory, f'galaxy_{galaxy_id_tag}.condor.desc')
executable = cjs.job_file
build_submit_params = dict(
@@ -117,7 +117,7 @@ class CondorJobRunner(AsynchronousJobRunner):
self.write_executable_script(executable, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception("(%s) failure preparing job script" % galaxy_id_tag)
log.exception(f"({galaxy_id_tag}) failure preparing job script")
return
cleanup_job = job_wrapper.cleanup_job
@@ -128,7 +128,7 @@ class CondorJobRunner(AsynchronousJobRunner):
cjs.cleanup()
# job_wrapper.fail() calls job_wrapper.cleanup()
job_wrapper.fail("failure preparing submit file", exception=True)
log.exception("(%s) failure preparing submit file" % galaxy_id_tag)
log.exception(f"({galaxy_id_tag}) failure preparing submit file")
return
# job was deleted while we were preparing it
@@ -274,7 +274,7 @@ class CondorJobRunner(AsynchronousJobRunner):
cjs.command_line = job.get_command_line()
cjs.job_wrapper = job_wrapper
cjs.job_destination = job_wrapper.job_destination
cjs.user_log = os.path.join(job_wrapper.working_directory, 'galaxy_%s.condor.log' % galaxy_id_tag)
cjs.user_log = os.path.join(job_wrapper.working_directory, f'galaxy_{galaxy_id_tag}.condor.log')
cjs.register_cleanup_file_attribute('user_log')
if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED):
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the DRM queue")
+19 -19
View File
@@ -45,8 +45,8 @@ class DRMAAJobRunner(AsynchronousJobRunner):
runner_param_specs = {
'drmaa_library_path': dict(map=str, default=os.environ.get('DRMAA_LIBRARY_PATH', None))}
for retry_exception in RETRY_EXCEPTIONS_LOWER:
runner_param_specs[retry_exception + '_state'] = dict(map=str, valid=lambda x: x in (model.Job.states.OK, model.Job.states.ERROR), default=model.Job.states.OK)
runner_param_specs[retry_exception + '_retries'] = dict(map=int, valid=lambda x: int(x) >= 0, default=0)
runner_param_specs[f"{retry_exception}_state"] = dict(map=str, valid=lambda x: x in (model.Job.states.OK, model.Job.states.ERROR), default=model.Job.states.OK)
runner_param_specs[f"{retry_exception}_retries"] = dict(map=int, valid=lambda x: int(x) >= 0, default=0)
if 'runner_param_specs' not in kwargs:
kwargs['runner_param_specs'] = dict()
@@ -109,7 +109,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
log.debug(f"Converted URL '{url}' to destination runner=drmaa, params={params}")
return JobDestination(runner='drmaa', params=params)
else:
log.debug("Converted URL '%s' to destination runner=drmaa" % url)
log.debug(f"Converted URL '{url}' to destination runner=drmaa")
return JobDestination(runner='drmaa')
def get_native_spec(self, url):
@@ -144,8 +144,8 @@ class DRMAAJobRunner(AsynchronousJobRunner):
remoteCommand=ajs.job_file,
jobName=ajs.job_name,
workingDirectory=job_wrapper.working_directory,
outputPath=":%s" % ajs.output_file,
errorPath=":%s" % ajs.error_file
outputPath=f":{ajs.output_file}",
errorPath=f":{ajs.error_file}"
)
# Avoid a jt.exitCodePath for now - it's only used when finishing.
@@ -161,7 +161,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception("(%s) failure writing job script" % galaxy_id_tag)
log.exception(f"({galaxy_id_tag}) failure writing job script")
return
# job was deleted while we were preparing it
@@ -195,7 +195,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
log.exception('(%s) drmaa.Session.runJob() failed unconditionally', galaxy_id_tag)
trynum = 5
else:
log.error("(%s) All attempts to submit job failed" % galaxy_id_tag)
log.error(f"({galaxy_id_tag}) All attempts to submit job failed")
if not fail_msg:
fail_msg = DEFAULT_JOB_PUT_FAILURE_MESSAGE
job_wrapper.fail(fail_msg)
@@ -207,16 +207,16 @@ class DRMAAJobRunner(AsynchronousJobRunner):
pwent = job_wrapper.user_system_pwent
if pwent is None:
if not allow_guests:
fail_msg = "User %s is not mapped to any real user, and not permitted to start jobs." % job_wrapper.user
fail_msg = f"User {job_wrapper.user} is not mapped to any real user, and not permitted to start jobs."
job_wrapper.fail(fail_msg)
return
pwent = job_wrapper.galaxy_system_pwent
log.debug('({}) submitting with credentials: {} [uid: {}]'.format(galaxy_id_tag, pwent[0], pwent[2]))
log.debug(f'({galaxy_id_tag}) submitting with credentials: {pwent[0]} [uid: {pwent[2]}]')
filename = self.store_jobtemplate(job_wrapper, jt)
self.userid = pwent[2]
external_job_id = self.external_runjob(external_runjob_script, filename, pwent[2])
if external_job_id is None:
job_wrapper.fail("(%s) could not queue job" % galaxy_id_tag)
job_wrapper.fail(f"({galaxy_id_tag}) could not queue job")
return
log.info(f"({galaxy_id_tag}) queued as {external_job_id}")
@@ -283,11 +283,11 @@ class DRMAAJobRunner(AsynchronousJobRunner):
state = self.ds.job_status(external_job_id)
# Reset exception retries
for retry_exception in RETRY_EXCEPTIONS_LOWER:
setattr(ajs, retry_exception + '_retries', 0)
setattr(ajs, f"{retry_exception}_retries", 0)
except (drmaa.InternalException, drmaa.InvalidJobException) as e:
ecn = type(e).__name__
retry_param = ecn.lower() + '_retries'
state_param = ecn.lower() + '_state'
retry_param = f"{ecn.lower()}_retries"
state_param = f"{ecn.lower()}_state"
retries = getattr(ajs, retry_param, 0)
log.warning("(%s/%s) unable to check job status because of %s exception for %d consecutive tries: %s", galaxy_id_tag, external_job_id, ecn, retries + 1, e)
if self.runner_params[retry_param] > 0:
@@ -332,7 +332,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
if state is None:
continue
if state != old_state:
log.debug("({}/{}) state change: {}".format(galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state]))
log.debug(f"({galaxy_id_tag}/{external_job_id}) state change: {self.drmaa_job_state_strings[state]}")
if state == drmaa.JobState.RUNNING and not ajs.running:
ajs.running = True
ajs.job_wrapper.change_state(model.Job.states.RUNNING)
@@ -370,7 +370,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
except drmaa.InvalidJobException:
log.exception(f"({job.id}/{ext_id}) User killed running job, but it was already dead")
except commands.CommandLineException as e:
log.error("({}/{}) User killed running job, but command execution failed: {}".format(job.id, ext_id, unicodify(e)))
log.error(f"({job.id}/{ext_id}) User killed running job, but command execution failed: {unicodify(e)}")
except Exception:
log.exception(f"({job.id}/{ext_id}) User killed running job, but error encountered removing from DRM queue")
@@ -431,12 +431,12 @@ class DRMAAJobRunner(AsynchronousJobRunner):
galaxy_id_tag = job_wrapper.get_id_tag()
# define job attributes
job_name = 'g%s' % galaxy_id_tag
job_name = f'g{galaxy_id_tag}'
if job_wrapper.tool.old_id:
job_name += '_%s' % job_wrapper.tool.old_id
job_name += f'_{job_wrapper.tool.old_id}'
if not self.redact_email_in_job_name and external_runjob_script is None:
job_name += '_%s' % job_wrapper.user
job_name = ''.join(x if x in (string.ascii_letters + string.digits + '_') else '_' for x in job_name)
job_name += f'_{job_wrapper.user}'
job_name = ''.join(x if x in (f"{string.ascii_letters + string.digits}_") else '_' for x in job_name)
if self.restrict_job_name_length:
job_name = job_name[:self.restrict_job_name_length]
return job_name
+18 -18
View File
@@ -157,7 +157,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
log.error("Job creation failure. No Response from GoDocker")
job_wrapper.fail("Not submitted")
else:
log.debug("Starting queue_job for job " + job_id)
log.debug(f"Starting queue_job for job {job_id}")
# Create an object of AsynchronousJobState and add it to the monitor queue.
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, job_wrapper=job_wrapper, job_id=job_id, job_destination=job_destination)
self.monitor_queue.put(ajs)
@@ -180,7 +180,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
""" Get task from GoDocker """
job_persisted_state = job_state.job_wrapper.get_state()
job_status_god = self.get_task(job_state.job_id)
log.debug("Job ID: " + str(job_state.job_id) + " Job Status: " + str(job_status_god['status']['primary']))
log.debug(f"Job ID: {str(job_state.job_id)} Job Status: {str(job_status_god['status']['primary'])}")
if job_status_god['status']['primary'] == "over" or job_persisted_state == model.Job.states.STOPPED:
job_state.running = False
@@ -234,7 +234,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
No Return data expected
'''
job_id = job_wrapper.job_id
log.debug("STOP JOB EXECUTION OF JOB ID: " + str(job_id))
log.debug(f"STOP JOB EXECUTION OF JOB ID: {str(job_id)}")
# Get task status from GoDocker.
job_status_god = self.get_task_status(job_id)
if job_status_god['status']['primary'] != "over":
@@ -276,8 +276,8 @@ class GodockerJobRunner(AsynchronousJobRunner):
if vol['name'] == "go-docker":
path = str(vol['path'])
if path:
god_output_file = path + "/god.log"
god_error_file = path + "/god.err"
god_output_file = f"{path}/god.log"
god_error_file = f"{path}/god.err"
try:
# Read from GoDocker output_file and write it into galaxy output_file.
f = open(god_output_file)
@@ -299,9 +299,9 @@ class GodockerJobRunner(AsynchronousJobRunner):
log_file.write(out_log)
log_file.close()
f.close()
log.debug("CREATE OUTPUT FILE: " + job_state.output_file)
log.debug("CREATE ERROR FILE: " + job_state.error_file)
log.debug("CREATE EXIT CODE FILE: " + job_state.exit_code_file)
log.debug(f"CREATE OUTPUT FILE: {job_state.output_file}")
log.debug(f"CREATE ERROR FILE: {job_state.error_file}")
log.debug(f"CREATE EXIT CODE FILE: {job_state.exit_code_file}")
except OSError as e:
log.error('Could not access task log file: %s', unicodify(e))
log.debug("IO Error occurred when accessing the files.")
@@ -315,7 +315,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
Create Login model schema of GoDocker and call the http_post_request method.
"""
log.debug("LOGIN TASK TO BE EXECUTED \n")
log.debug("GODOCKER LOGIN: " + str(login))
log.debug(f"GODOCKER LOGIN: {str(login)}")
data = json.dumps({'user': login, 'apikey': apikey})
# Create object of Godocker class
g_auth = Godocker(server, login, apikey, noCert)
@@ -348,7 +348,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
docker_image = self._find_container(job_wrapper).container_id
log.debug("GoDocker runner using container %s.", docker_image)
except Exception:
log.error("Unable to find docker_image for job %s, failing." % job_wrapper.job_id)
log.error(f"Unable to find docker_image for job {job_wrapper.job_id}, failing.")
return False
volumes = []
@@ -378,11 +378,11 @@ class GodockerJobRunner(AsynchronousJobRunner):
if(job_destination.params["virtualenv"] == "true"):
GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;"""
venv = GALAXY_VENV_TEMPLATE % job_wrapper.galaxy_virtual_env
command = "#!/bin/bash\n" + "cd " + job_wrapper.working_directory + "\n" + venv + "\n" + job_wrapper.runner_command_line
command = f"#!/bin/bash\ncd {job_wrapper.working_directory}\n{venv}\n{job_wrapper.runner_command_line}"
else:
command = "#!/bin/bash\n" + "cd " + job_wrapper.working_directory + "\n" + job_wrapper.runner_command_line
command = f"#!/bin/bash\ncd {job_wrapper.working_directory}\n{job_wrapper.runner_command_line}"
except Exception:
command = "#!/bin/bash\n" + "cd " + job_wrapper.working_directory + "\n" + job_wrapper.runner_command_line
command = f"#!/bin/bash\ncd {job_wrapper.working_directory}\n{job_wrapper.runner_command_line}"
# GoDocker Job model schema
job = {
@@ -424,7 +424,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
result = self.auth.http_post_request(
"/api/1.0/task", json.dumps(job),
{'Authorization': 'Bearer ' + self.auth.token, 'Content-type': 'application/json', 'Accept': 'application/json'}
{'Authorization': f"Bearer {self.auth.token}", 'Content-type': 'application/json', 'Accept': 'application/json'}
)
# Return job_id
return str(result.json()['id'])
@@ -435,7 +435,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
"""
job = False
if self.auth.token:
result = self.auth.http_get_request("/api/1.0/task/" + str(job_id), {'Authorization': 'Bearer ' + self.auth.token})
result = self.auth.http_get_request(f"/api/1.0/task/{str(job_id)}", {'Authorization': f"Bearer {self.auth.token}"})
job = result.json()
# Return the job
return job
@@ -446,7 +446,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
"""
job = False
if self.auth.token:
result = self.auth.http_get_request("/api/1.0/task/" + str(job_id) + "/suspend", {'Authorization': 'Bearer ' + self.auth.token})
result = self.auth.http_get_request(f"/api/1.0/task/{str(job_id)}/suspend", {'Authorization': f"Bearer {self.auth.token}"})
job = result.json()
# Return the job
return job
@@ -457,7 +457,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
"""
job = False
if self.auth.token:
result = self.auth.http_get_request("/api/1.0/task/" + str(job_id) + "/status", {'Authorization': 'Bearer ' + self.auth.token})
result = self.auth.http_get_request(f"/api/1.0/task/{str(job_id)}/status", {'Authorization': f"Bearer {self.auth.token}"})
job = result.json()
# Return task status
return job
@@ -468,7 +468,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
"""
job = False
if self.auth.token:
result = self.auth.http_delete_request("/api/1.0/task/" + str(job_id), {'Authorization': 'Bearer ' + self.auth.token})
result = self.auth.http_delete_request(f"/api/1.0/task/{str(job_id)}", {'Authorization': f"Bearer {self.auth.token}"})
job = result.json()
# Return the job
return job
+12 -12
View File
@@ -127,7 +127,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug("Starting queue_job for job " + job_wrapper.get_id_tag())
log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
@@ -144,7 +144,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception("(%s) failure writing job script" % job_wrapper.get_id_tag())
log.exception(f"({job_wrapper.get_id_tag()}) failure writing job script")
return
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
@@ -240,7 +240,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
"""
label_val = self.LABEL_REGEX.sub("_", value)
if not self.LABEL_START.search(label_val):
label_val = 'x' + label_val
label_val = f"x{label_val}"
if not self.LABEL_END.search(label_val):
label_val += 'x'
return label_val
@@ -308,7 +308,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
}
},
"spec": {
"ports": [{"name": "job-{}-{}".format(self.__force_label_conformity(ajs.job_wrapper.get_id_tag()), p),
"ports": [{"name": f"job-{self.__force_label_conformity(ajs.job_wrapper.get_id_tag())}-{p}",
"port": int(p),
"protocol": "TCP",
"targetPort": int(p)} for p in guest_ports],
@@ -521,14 +521,14 @@ class KubernetesJobRunner(AsynchronousJobRunner):
repo = ""
owner = ""
if 'repo' in job_destination.params:
repo = job_destination.params['repo'] + "/"
repo = f"{job_destination.params['repo']}/"
if 'owner' in job_destination.params:
owner = job_destination.params['owner'] + "/"
owner = f"{job_destination.params['owner']}/"
k8s_cont_image = repo + owner + job_destination.params['image']
if 'tag' in job_destination.params:
k8s_cont_image += ":" + job_destination.params['tag']
k8s_cont_image += f":{job_destination.params['tag']}"
return k8s_cont_image
@@ -538,12 +538,12 @@ class KubernetesJobRunner(AsynchronousJobRunner):
if isinstance(raw_id, str):
cleaned_id = re.sub("[^-a-z0-9]", "-", raw_id)
if cleaned_id.startswith("-") or cleaned_id.endswith("-"):
cleaned_id = "x%sx" % cleaned_id
cleaned_id = f"x{cleaned_id}x"
return cleaned_id
return "job-container"
def __get_k8s_job_name(self, prefix, job_wrapper):
return "{}-{}".format(prefix, self.__force_label_conformity(job_wrapper.get_id_tag()))
return f"{prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}"
def __get_destination_params(self, job_wrapper):
"""Obtains allowable runner param overrides from the destination"""
@@ -719,7 +719,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
return any(True for c in conditions if c['type'] == 'Failed' and c['reason'] == 'DeadlineExceeded')
def _get_pod_for_job(self, job_state):
pods = Pod.objects(self._pykube_api).filter(selector="app=%s" % job_state.job_id,
pods = Pod.objects(self._pykube_api).filter(selector=f"app={job_state.job_id}",
namespace=self.runner_params['k8s_namespace'])
if not pods.response['items']:
return None
@@ -764,7 +764,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
k8s_job = Job(self._pykube_api, job_to_delete.response['items'][0])
if job_wrapper.guest_ports:
k8s_job_prefix = self.__produce_k8s_job_prefix()
k8s_job_name = "{}-{}".format(k8s_job_prefix, self.__force_label_conformity(job_wrapper.get_id_tag()))
k8s_job_name = f"{k8s_job_prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}"
log.debug(f'Deleting service/ingress for job with ID {job_wrapper.get_id_tag()}')
job_failed = (k8s_job.obj['status']['failed'] > 0
if 'failed' in k8s_job.obj['status'] else False)
@@ -788,7 +788,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
def recover(self, job, job_wrapper):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
job_id = job.get_job_runner_external_id()
log.debug("k8s trying to recover job: " + job_id)
log.debug(f"k8s trying to recover job: {job_id}")
if job_id is None:
self.put(job_wrapper)
return

Some files were not shown because too many files have changed in this diff Show More