diff --git a/lib/galaxy/containers/docker.py b/lib/galaxy/containers/docker.py index c61622fbb23..a246bf75307 100644 --- a/lib/galaxy/containers/docker.py +++ b/lib/galaxy/containers/docker.py @@ -276,12 +276,12 @@ class DockerAPIClient(object): if tries > 1: log.info('%s() succeeded on attempt %s', qualname, tries) return r - except ConnectionError as exc: + except ConnectionError: reinit = True except docker.errors.APIError as exc: if not DockerAPIClient._should_retry_request(exc.response.status_code): raise - except ReadTimeout as exc: + except ReadTimeout: reinit = True retry_time = 0 finally: diff --git a/lib/galaxy/containers/docker_model.py b/lib/galaxy/containers/docker_model.py index 699022019eb..23e41f5b5f5 100644 --- a/lib/galaxy/containers/docker_model.py +++ b/lib/galaxy/containers/docker_model.py @@ -189,7 +189,7 @@ class DockerService(Container): def from_cli(cls, interface, s, task_list): service = cls(interface, s['ID'], name=s['NAME'], image=s['IMAGE']) for task_dict in task_list: - if task_dict['NAME'].strip().startswith('\_'): + if task_dict['NAME'].strip().startswith(r'\_'): continue # historical task service.task_add(DockerTask.from_cli(interface, task_dict, service=service)) return service diff --git a/lib/galaxy/containers/docker_swarm.py b/lib/galaxy/containers/docker_swarm.py index 51a31fe8d1d..27e21c5d607 100644 --- a/lib/galaxy/containers/docker_swarm.py +++ b/lib/galaxy/containers/docker_swarm.py @@ -280,7 +280,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): def service_tasks(self, service): for task_dict in self.service_ps(service.id): - if task_dict['NAME'].strip().startswith('\_'): + if task_dict['NAME'].strip().startswith(r'\_'): continue # historical task yield DockerTask.from_cli(self, task_dict, service=service) diff --git a/lib/galaxy/datatypes/assembly.py b/lib/galaxy/datatypes/assembly.py index 64b5662c092..4626c5b3ed3 100644 --- a/lib/galaxy/datatypes/assembly.py +++ b/lib/galaxy/datatypes/assembly.py @@ -174,15 +174,15 @@ class Velvet(Html): log_path = os.path.join(efp, 'Log') with open(log_path, 'r') as f: log_content = f.read(1000) - log_msg = re.sub('/\S*/', '', log_content) + log_msg = re.sub(r'/\S*/', '', log_content) log.debug("Velveth log info %s" % log_msg) - paired_end_reads = re.search('-(short|long)Paired', log_msg) is not None + paired_end_reads = re.search(r'-(short|long)Paired', log_msg) is not None dataset.metadata.paired_end_reads = paired_end_reads - long_reads = re.search('-long', log_msg) is not None + long_reads = re.search(r'-long', log_msg) is not None dataset.metadata.long_reads = long_reads - short2_reads = re.search('-short(Paired)?2', log_msg) is not None + short2_reads = re.search(r'-short(Paired)?2', log_msg) is not None dataset.metadata.short2_reads = short2_reads - dataset.info = re.sub('.*velveth \S+', 'hash_length', re.sub('\n', ' ', log_msg)) + dataset.info = re.sub(r'.*velveth \S+', 'hash_length', re.sub(r'\n', ' ', log_msg)) if paired_end_reads: gen_msg = gen_msg + ' Paired-End Reads' if long_reads: diff --git a/lib/galaxy/datatypes/genetics.py b/lib/galaxy/datatypes/genetics.py index 5563fdf9980..ab22d49e13d 100644 --- a/lib/galaxy/datatypes/genetics.py +++ b/lib/galaxy/datatypes/genetics.py @@ -32,8 +32,8 @@ gal_Log = logging.getLogger(__name__) verbose = False # https://genome.ucsc.edu/goldenpath/help/hgGenomeHelp.html -VALID_GENOME_GRAPH_MARKERS = re.compile('^(chr.*|RH.*|rs.*|SNP_.*|CN.*|A_.*)') -VALID_GENOTYPES_LINE = re.compile('^([a-zA-Z0-9]+)(\\s([0-9]{2}|[A-Z]{2}|NC|\?\?))+\\s*$') +VALID_GENOME_GRAPH_MARKERS = re.compile(r'^(chr.*|RH.*|rs.*|SNP_.*|CN.*|A_.*)') +VALID_GENOTYPES_LINE = re.compile(r'^([a-zA-Z0-9]+)(\s([0-9]{2}|[A-Z]{2}|NC|\?\?))+\s*$') @build_sniff_from_prefix diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py index 7b1cfa6f44a..0089fd326f6 100644 --- a/lib/galaxy/datatypes/isa.py +++ b/lib/galaxy/datatypes/isa.py @@ -2,11 +2,10 @@ ISA datatype See https://github.com/ISA-tools - """ - from __future__ import print_function +import io import json import logging import os @@ -16,14 +15,14 @@ import shutil import sys import tempfile from cgi import escape -from json import dumps # noqa: F401 # Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail. logging.getLogger("isatools.isatab").setLevel(logging.ERROR) -from isatools import isajson -from isatools import isatab_meta +from isatools import ( + isajson, + isatab_meta +) -from galaxy import model from galaxy import util from galaxy.datatypes import data from galaxy.util.compression_utils import CompressedFile @@ -89,22 +88,9 @@ class _Isa(data.Data): ################################################################ def _get_isa_folder_path(self, dataset): - - isa_folder = None - - if dataset: - if isinstance(dataset, model.Dataset): - isa_folder = dataset.extra_files_path - if isinstance(dataset, model.HistoryDatasetAssociation): - # XXX With this loop the dataset name is reset inside the history to the ISA archive ID. Why? - for attr, value in dataset.__dict__.iteritems(): - if str(attr) == '_metadata_collection': - datatype = value.parent.datatype # noqa: F841 - isa_folder = dataset.dataset.extra_files_path - - if isa_folder is None: + isa_folder = dataset.extra_files_path + if not isa_folder: raise Exception('Unvalid dataset object, or no extra files path found for this dataset.') - return isa_folder # Get main file {{{2 @@ -176,7 +162,7 @@ class _Isa(data.Data): raise RuntimeError("Unable to find the main file within the 'files_path' folder") # Read first lines of main file - with open(main_file, "r") as f: + with io.open(main_file, encoding='utf-8') as f: data = [] for line in f: if len(data) < _MAX_LINES_HISTORY_PEEK: diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index 8d29e58f4d1..fc5b1ceb1fb 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -45,7 +45,7 @@ def count_lines(filename, non_empty=False): """ try: if non_empty: - out = subprocess.Popen(['grep', '-cve', '^\s*$', filename], stdout=subprocess.PIPE) + out = subprocess.Popen(['grep', '-cve', r'^\s*$', filename], stdout=subprocess.PIPE) else: out = subprocess.Popen(['wc', '-l', filename], stdout=subprocess.PIPE) return int(out.communicate()[0].split()[0]) @@ -136,7 +136,7 @@ class SDF(GenericMolFile): """ Set the number of molecules in dataset. """ - dataset.metadata.number_of_molecules = count_special_lines("^\$\$\$\$$", dataset.file_name) + dataset.metadata.number_of_molecules = count_special_lines(r"^\$\$\$\$$", dataset.file_name) def split(cls, input_datasets, subdir_generator_function, split_params): """ @@ -717,7 +717,7 @@ class CML(GenericXml): """ Set the number of lines of data in dataset. """ - dataset.metadata.number_of_molecules = count_special_lines('^\s*') and re.search("[\(\)\[\]\.]", line): + if not line.startswith('>') and re.search(r"[\(\)\[\]\.]", line): break return True else: @@ -529,7 +529,7 @@ class csFasta(Sequence): break elif line[0] not in string.ascii_uppercase: return False - elif len(line) > 1 and not re.search('^[\d.]+$', line[1:]): + elif len(line) > 1 and not re.search(r'^[\d.]+$', line[1:]): return False return True else: @@ -1013,8 +1013,8 @@ class DotBracket(Sequence): edam_format = "format_1457" file_ext = "dbn" - sequence_regexp = re.compile("^[ACGTURYKMSWBDHVN]+$", re.I) - structure_regexp = re.compile("^[\(\)\.\[\]{}]+$") + sequence_regexp = re.compile(r"^[ACGTURYKMSWBDHVN]+$", re.I) + structure_regexp = re.compile(r"^[\(\)\.\[\]{}]+$") def set_meta(self, dataset, **kwd): """ diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 9e5c894cefb..7c67dfc51e3 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1124,12 +1124,12 @@ class ConnectivityTable(Tabular): if not self.header_regexp.match(line): return False else: - length = int(re.split('\W+', line, 1)[0]) + length = int(re.split(r'\W+', line, 1)[0]) else: if not self.structure_regexp.match(line.upper()): return False else: - if j != int(re.split('\W+', line, 1)[0]): + if j != int(re.split(r'\W+', line, 1)[0]): return False elif j == length: # Last line of first sequence has been recheached return True diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 35420fdd98a..44c2dd56fbc 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -416,7 +416,7 @@ class SnpEffDb(Text): with gzip.open(path, 'rb') as fh: buf = fh.read(100) lines = buf.splitlines() - m = re.match('^(SnpEff)\s+(\d+\.\d+).*$', lines[0].strip()) + m = re.match(r'^(SnpEff)\s+(\d+\.\d+).*$', lines[0].strip()) if m: snpeff_version = m.groups()[0] + m.groups()[1] except Exception: diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 421a5886c75..c130441754c 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -217,7 +217,7 @@ class GFFReaderWrapper(NiceReaderWrapper): try: interval = super(GenomicIntervalReader, self).__next__() raw_size += len(self.current_line) - except StopIteration as e: + except StopIteration: # No more intervals to read, but last feature needs to be # returned. interval = None diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index 753f31b6450..1a7b2bf7218 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -39,7 +39,7 @@ class GenericXml(data.Text): if line is None or not line.startswith('= '2.6' +twill==0.9.1 ; python_version < '3' +typing==3.6.6 ; python_version < '3.5' +urllib3==1.24 watchdog==0.9.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index dfb5052f0c1..e8f3fc253ea 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -1,6 +1,6 @@ -i https://wheels.galaxyproject.org/simple --extra-index-url https://pypi.python.org/simple -adal==1.1.0 +adal==1.2.0 amqp==2.3.2 appdirs==1.4.3 asn1crypto==0.24.0 @@ -10,56 +10,56 @@ azure-cosmosdb-table==1.0.4 azure-mgmt-compute==4.0.1 azure-mgmt-devtestlabs==2.2.0 azure-mgmt-network==2.1.0 -azure-mgmt-nspkg==2.0.0 +azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==2.0.0 azure-mgmt-storage==2.0.0 -azure-nspkg==2.0.0 +azure-nspkg==3.0.2 azure-storage-blob==1.3.1 azure-storage-common==1.3.0 azure-storage-nspkg==3.0.0 babel==2.6.0 -bagit==1.6.4 +bagit==1.7.0 bcrypt==3.1.4 -bdbag==1.4.1 +bdbag==1.5.0 beaker==1.10.0 bioblend==0.11.0 -bleach==2.1.4 +bleach==3.0.2 boltons==18.0.1 boto3==1.7.84 boto==2.49.0 botocore==1.10.84 bunch==1.0.1 bx-python==0.8.2 -bz2file==0.98; python_version < '3.3' -certifi==2018.8.24 +bz2file==0.98 ; python_version < '3.3' +cachetools==2.1.0 +certifi==2018.10.15 cffi==1.11.5 chardet==3.0.4 cheetah3==3.1.0 -cliff==2.13.0; python_version >= '2.6' +cliff==2.13.0 cloudauthz==0.2.0 -cloudbridge==1.0.1 +cloudbridge==1.0.2 cmd2==0.8.9 -contextlib2==0.5.5; python_version < '3.5' +contextlib2==0.5.5 ; python_version < '3.5' cryptography==2.3.1 debtcollector==1.20.0 decorator==4.3.0 -deprecation==2.0.5 +deprecation==2.0.6 dictobj==0.4 docopt==0.6.2 docutils==0.14 dogpile.cache==0.6.7 -enum34==1.1.6; python_version < '3.4' +enum34==1.1.6 ; python_version < '3.4' fabric3==1.14.post1 -funcsigs==1.0.2; python_version < '3.3' -functools32==3.2.3.post2; python_version == '2.7' +funcsigs==1.0.2 ; python_version < '3.3' +functools32==3.2.3.post2 ; python_version == '2.7' future==0.16.0 -futures==3.2.0; python_version == '2.6' or python_version == '2.7' +futures==3.2.0 ; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.3 -gxformat2==0.6.0 +gxformat2==0.7.1 h5py==2.8.0 -html5lib==1.0.1 idna==2.7 -ipaddress==1.0.22; python_version < '3.3' +ipaddress==1.0.22 ; python_version < '3.3' isa-rwval==0.10.7 iso8601==0.1.12 isodate==0.6.0 @@ -71,8 +71,8 @@ keystoneauth1==3.11.0 kombu==4.2.1 mako==1.0.7 markupsafe==1.0 -mercurial==3.7.3; python_version < '3' -monotonic==1.5; python_version == '2.7' +mercurial==3.7.3 ; python_version < '3' +monotonic==1.5 msgpack==0.5.6 msrest==0.5.5 msrestazure==0.5.0 @@ -82,44 +82,44 @@ netifaces==0.10.7 networkx==1.11 nodeenv==1.3.2 nose==1.3.7 -numpy==1.15.1 +numpy==1.15.3 oauthlib==2.1.0 -openstacksdk==0.17.0; python_version >= '2.6' -os-client-config==1.31.2; python_version >= '2.6' +openstacksdk==0.17.0 +os-client-config==1.31.2 os-service-types==1.3.0 osc-lib==1.11.1 -oslo.config==6.4.0 +oslo.config==6.6.2 oslo.context==2.21.0 -oslo.i18n==3.21.0 -oslo.log==3.39.0 -oslo.serialization==2.27.0; python_version != '3.3.*' -oslo.utils==3.37.0; python_version != '3.3.*' -packaging==17.1 +oslo.i18n==3.22.1 +oslo.log==3.40.1 +oslo.serialization==2.28.1 +oslo.utils==3.37.1 +packaging==18.0 paramiko==2.4.2 parsley==1.3 -paste==2.0.3 +paste==3.0.0 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.2.0 +pbr==5.1.0 prettytable==0.7.2 psutil==5.4.7 pulsar-galaxy-lib==0.8.3 pyasn1==0.4.4 -pycparser==2.18 +pycparser==2.19 pycryptodome==3.6.6 pycryptodomex==3.6.6 -pyinotify==0.9.6; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' -pyjwkest==1.4.0 +pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' +pyjwkest==1.4.0 ; extra == 'openidconnect' pyjwt==1.6.4 -pykwalify==1.6.1 -pynacl==1.2.1 +pykwalify==1.7.0 +pynacl==1.3.0 pyopenssl==18.0.0 -pyparsing==2.2.0 -pyperclip==1.6.4 +pyparsing==2.2.2 +pyperclip==1.7.0 pysam==0.14.1 pysftp==0.2.9 python-cinderclient==4.0.0 -python-dateutil==2.7.3; python_version >= '2.7' +python-dateutil==2.7.3 python-genomespaceclient==1.1.0 python-glanceclient==2.12.0 python-keystoneclient==3.17.0 @@ -127,37 +127,37 @@ python-neutronclient==6.9.0 python-novaclient==11.0.0 python-openid==2.2.5 python-swiftclient==3.6.0 -pytz==2018.5 +pytz==2018.6 pyyaml==3.13 repoze.lru==0.7 requests-oauthlib==1.0.0 requests-toolbelt==0.8.0 -requests==2.19.1 +requests==2.20.0 requestsexceptions==1.4.0 rfc3986==1.1.0 routes==2.4.1 s3transfer==0.1.13 simplejson==3.16.0 six==1.11.0 -social-auth-core==1.5.0 +social-auth-core[openidconnect]==1.5.0 sqlalchemy-migrate==0.11.0 -sqlalchemy-utils==0.33.4 -sqlalchemy==1.2.11 +sqlalchemy-utils==0.33.6 +sqlalchemy==1.2.12 sqlparse==0.2.4 -stevedore==1.29.0 -subprocess32==3.5.2; python_version < '3.0' +stevedore==1.30.0 +subprocess32==3.5.3 ; python_version < '3.0' svgwrite==1.1.12 tempita==0.5.2 -tenacity==4.12.0; python_version >= '2.6' -typing==3.6.6; python_version < '3.5' +tenacity==4.12.0 +typing==3.6.6 ; python_version < '3.5' tzlocal==1.5.1 -unicodecsv==0.14.1; python_version < '3.0' -urllib3==1.23; python_version >= '2.6' +unicodecsv==0.14.1 ; python_version < '3.0' +urllib3==1.24 uwsgi==2.0.17.1 vine==1.1.4 -warlock==1.3.0; python_version != '3.3.*' -wcwidth==0.1.7; sys_platform != 'win32' +warlock==1.3.0 +wcwidth==0.1.7 ; sys_platform != 'win32' webencodings==0.5.1 -webob==1.8.2; python_version != '3.2.*' +webob==1.8.3 whoosh==2.7.4 wrapt==1.10.11 diff --git a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt index b4222caa469..ee4270d1559 100644 --- a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt @@ -1,8 +1,8 @@ -i https://pypi.python.org/simple -configparser==3.5.0; python_version < '3.2' -enum34==1.1.6; python_version < '3.4' +configparser==3.5.0 ; python_version < '3.2' +enum34==1.1.6 ; python_version < '3.4' flake8-import-order==0.18 -flake8==3.5.0 +flake8==3.6.0 mccabe==0.6.1 -pycodestyle==2.3.1 -pyflakes==1.6.0 +pycodestyle==2.4.0 +pyflakes==2.0.0 diff --git a/lib/galaxy/dependencies/pipfiles/update.sh b/lib/galaxy/dependencies/pipfiles/update.sh index ff6f62920d4..5dbe6ca7b22 100755 --- a/lib/galaxy/dependencies/pipfiles/update.sh +++ b/lib/galaxy/dependencies/pipfiles/update.sh @@ -42,14 +42,16 @@ for env in $ENVS; do pipenv lock -r | sed -e 's/--hash[^[:space:]]*//g' -e 's/[[:space:]]*$//' > pinned-requirements.txt pipenv lock -r --dev | sed -e 's/--hash[^[:space:]]*//g' -e 's/[[:space:]]*$//' > pinned-dev-requirements.txt # Fix oscillating environment markers - sed -i.orig -e "s/^cffi==\([^;]\{1,\}\).*$/cffi==\1/" \ - -e "s/^cmd2==\([^;]\{1,\}\).*$/cmd2==\1/" \ - -e "s/^enum34==\([^;]\{1,\}\).*$/enum34==\1; python_version < '3.4'/" \ - -e "s/^funcsigs==\([^;]\{1,\}\).*$/funcsigs==\1; python_version < '3.3'/" \ - -e "s/^futures==\([^;]\{1,\}\).*$/futures==\1; python_version == '2.6' or python_version == '2.7'/" \ - -e "s/^py2-ipaddress==\([^;]\{1,\}\).*$/py2-ipaddress==\1; python_version < '3'/" \ - -e "s/^pyinotify==\([^;]\{1,\}\).*$/pyinotify==\1; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'/" \ - -e "s/^subprocess32==\([^;]\{1,\}\).*$/subprocess32==\1; python_version < '3.0'/" \ + sed -i.orig -e "s/^cffi==\([^ ;]\{1,\}\).*$/cffi==\1/" \ + -e "s/^cmd2==\([^ ;]\{1,\}\).*$/cmd2==\1/" \ + -e "s/^enum34==\([^ ;]\{1,\}\).*$/enum34==\1 ; python_version < '3.4'/" \ + -e "s/^funcsigs==\([^ ;]\{1,\}\).*$/funcsigs==\1 ; python_version < '3.3'/" \ + -e "s/^futures==\([^ ;]\{1,\}\).*$/futures==\1 ; python_version == '2.6' or python_version == '2.7'/" \ + -e "s/^monotonic==\([^ ;]\{1,\}\).*$/monotonic==\1/" \ + -e "s/^py2-ipaddress==\([^ ;]\{1,\}\).*$/py2-ipaddress==\1 ; python_version < '3'/" \ + -e "s/^pyinotify==\([^ ;]\{1,\}\).*$/pyinotify==\1 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'/" \ + -e "s/^python-dateutil==\([^ ;]\{1,\}\).*$/python-dateutil==\1/" \ + -e "s/^subprocess32==\([^ ;]\{1,\}\).*$/subprocess32==\1 ; python_version < '3.0'/" \ pinned-requirements.txt pinned-dev-requirements.txt done diff --git a/lib/galaxy/jobs/dynamic_tool_destination.py b/lib/galaxy/jobs/dynamic_tool_destination.py index b0e5ce8bb27..84f0525a87b 100755 --- a/lib/galaxy/jobs/dynamic_tool_destination.py +++ b/lib/galaxy/jobs/dynamic_tool_destination.py @@ -644,7 +644,7 @@ class RuleValidator(object): @return: validated rule and result of validation """ - emailregex = "^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$" + emailregex = r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$" if "users" in rule: if isinstance(rule["users"], list): diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 3195d8279a1..e4a999b9709 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -494,7 +494,7 @@ class JobHandlerQueue(Monitors): usage = self.app.quota_agent.get_usage(user=job.user, history=job.history) if usage > quota: return JOB_USER_OVER_QUOTA, job_destination - except AssertionError as e: + except AssertionError: pass # No history, should not happen with an anon user # Check total walltime limits if (state == JOB_READY and diff --git a/lib/galaxy/jobs/metrics/instrumenters/env.py b/lib/galaxy/jobs/metrics/instrumenters/env.py index 8bc962d2d01..a8c20455205 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/env.py +++ b/lib/galaxy/jobs/metrics/instrumenters/env.py @@ -48,7 +48,7 @@ class EnvPlugin(InstrumentPlugin): # We use '\n\}\n' as regex termination because shell # functions can be nested. # We use the non-greedy '.+?' because of re.DOTALL . - m = re.match('([^=]+)=(\(\) \{.+?\n\})\n', env_string, re.DOTALL) + m = re.match(r'([^=]+)=(\(\) \{.+?\n\})\n', env_string, re.DOTALL) if m is None: m = re.match('([^=]+)=(.*)\n', env_string) if m is None: diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 52a4c12c946..9203c05796b 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -188,7 +188,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): :rtype: """ if "k8s_galaxy_instance_id" in self.runner_params: - if re.match("(?!-)[a-z\d-]{1,20}(? GalaxyQuery(f,t,c) | ws -> GalaxyQuery(f, t, None) ) diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 7150c0831e8..7d26c7d5709 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -10,14 +10,14 @@ import re log = logging.getLogger(__name__) # Email validity parameters -VALID_EMAIL_RE = re.compile("[^@]+@[^@]+\.[^@]+") +VALID_EMAIL_RE = re.compile(r"[^@]+@[^@]+\.[^@]+") EMAIL_MAX_LEN = 255 # Public name validity parameters PUBLICNAME_MIN_LEN = 3 PUBLICNAME_MAX_LEN = 255 -VALID_PUBLICNAME_RE = re.compile("^[a-z0-9._\-]+$") -VALID_PUBLICNAME_SUB = re.compile("[^a-z0-9._\-]") +VALID_PUBLICNAME_RE = re.compile(r"^[a-z0-9._\-]+$") +VALID_PUBLICNAME_SUB = re.compile(r"[^a-z0-9._\-]") FILL_CHAR = '-' # Password validity parameters diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index bfbdfc812f5..583bb19286d 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1703,7 +1703,7 @@ class Tool(Dictifiable): tool_help = tool.help._source # Check each line of the rendered tool help for an image tag that points to a location under static/ for help_line in tool_help.split('\n'): - image_regex = re.compile('img alt="[^"]+" src="\${static_path}/([^"]+)"') + image_regex = re.compile(r'img alt="[^"]+" src="\${static_path}/([^"]+)"') matches = re.search(image_regex, help_line) if matches is not None: tool_help_image = matches.group(1) @@ -2760,7 +2760,7 @@ class RelabelFromFileTool(DatabaseOperationTool): dce_object = dce.element_object add_copied_value_to_new_elements(new_labels[i], dce_object) for key in new_elements.keys(): - if not re.match("^[\w\-_]+$", key): + if not re.match(r"^[\w\-_]+$", key): raise Exception("Invalid new colleciton identifier [%s]" % key) self._add_datasets_to_history(history, itervalues(new_elements)) output_collections.create_collection( diff --git a/lib/galaxy/tools/deps/brew_exts.py b/lib/galaxy/tools/deps/brew_exts.py index 9f6c0985c29..d5da7ebe70b 100755 --- a/lib/galaxy/tools/deps/brew_exts.py +++ b/lib/galaxy/tools/deps/brew_exts.py @@ -31,7 +31,7 @@ import string import subprocess import sys -WHITESPACE_PATTERN = re.compile("[\s]+") +WHITESPACE_PATTERN = re.compile(r"[\s]+") DESCRIPTION = "Script built on top of linuxbrew to operate on isolated, versioned brew installed environments." diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index d51bde35755..a43516e615d 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -39,7 +39,7 @@ SINGULARITY_CONTAINER_TYPE = "singularity" DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE] -LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = ''' +LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r''' python << EOF from __future__ import print_function @@ -57,7 +57,7 @@ cmd = "${images_cmd}" proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) stdo, stde = proc.communicate() found = False -for line in stdo.split("\\n"): +for line in stdo.split("\n"): tmp = re.split(r'\s+', line) if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value: found = True diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 716afb59685..b1f3af0bab1 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -1,7 +1,4 @@ -from galaxy.exceptions import ( - NotImplemented, - RequestParameterMissingException -) +from galaxy import exceptions class DependencyResolversView(object): @@ -34,7 +31,7 @@ class DependencyResolversView(object): requirements = [] resolver = self._dependency_resolver(index) if not hasattr(resolver, "list_dependencies"): - raise NotImplemented() + raise exceptions.NotImplemented() for requirement in resolver.list_dependencies(): requirements.append(requirement.to_dict()) return requirements @@ -128,7 +125,7 @@ class DependencyResolversView(object): """ resolver = self._dependency_resolver(index) if not hasattr(resolver, "install_dependency"): - raise NotImplemented() + raise exceptions.NotImplemented() name, version, type, extra_kwds = self._parse_dependency_info(payload) return resolver.install_dependency( @@ -157,7 +154,7 @@ class DependencyResolversView(object): extra_kwds = kwds.copy() name = extra_kwds.pop("name", None) if name is None: - raise RequestParameterMissingException("Missing 'name' parameter required for resolution.") + raise exceptions.RequestParameterMissingException("Missing 'name' parameter required for resolution.") version = extra_kwds.pop("version", None) type = extra_kwds.pop("type", "package") return name, version, type, extra_kwds @@ -227,7 +224,7 @@ class DependencyResolversView(object): if index: resolver = self._dependency_resolver(index) if not hasattr(resolver, "clean"): - raise NotImplemented() + raise exceptions.NotImplemented() else: resolver.clean() return "OK" diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index c20b0985f6b..d2cae015407 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -41,7 +41,7 @@ log = logging.getLogger(__name__) workflow_building_modes = Bunch(DISABLED=False, ENABLED=True, USE_HISTORY=1) -WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile('''\$\{.+?\}''') +WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile(r'\$\{.+?\}') def contains_workflow_parameter(value, search=False): @@ -529,7 +529,7 @@ class FileToolParameter(ToolParameter): # handle api upload session_id = value["session_id"] upload_store = trans.app.config.new_file_path - if re.match('^[\w-]+$', session_id) is None: + if re.match(r'^[\w-]+$', session_id) is None: raise ValueError("Invald session id format.") local_filename = os.path.abspath(os.path.join(upload_store, session_id)) else: diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 75a21ab8ccb..0e5bd18d8e0 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -744,8 +744,8 @@ class StdioParser(object): # Also note that whitespace is eliminated. # TODO: Turn this into a single match - it should be # more efficient. - code_range = re.sub("\s", "", code_range) - code_ranges = re.split(":", code_range) + code_range = re.sub(r"\s", "", code_range) + code_ranges = re.split(r":", code_range) if (len(code_ranges) == 2): if (code_ranges[0] is None or '' == code_ranges[0]): exit_code.range_start = float("-inf") @@ -827,8 +827,8 @@ class StdioParser(object): output_srcs = regex_elem.get("sources") if output_srcs is None: output_srcs = "output,error" - output_srcs = re.sub("\s", "", output_srcs) - src_list = re.split(",", output_srcs) + output_srcs = re.sub(r"\s", "", output_srcs) + src_list = re.split(r",", output_srcs) # Just put together anything to do with "out", including # "stdout", "output", etc. Repeat for "stderr", "error", # and anything to do with "err". If neither stdout nor diff --git a/lib/galaxy/tools/verify/asserts/__init__.py b/lib/galaxy/tools/verify/asserts/__init__.py index 59b90d32479..e43afde1b4a 100644 --- a/lib/galaxy/tools/verify/asserts/__init__.py +++ b/lib/galaxy/tools/verify/asserts/__init__.py @@ -21,7 +21,7 @@ for assertion_module_name in assertion_module_names: __import__(full_assertion_module_name) assertion_module = sys.modules[full_assertion_module_name] assertion_modules.append(assertion_module) - except Exception as e: + except Exception: log.exception('Failed to load assertion module: %s', assertion_module_name) diff --git a/lib/galaxy/tools/verify/test_data.py b/lib/galaxy/tools/verify/test_data.py index 305019dc770..db79d45b560 100644 --- a/lib/galaxy/tools/verify/test_data.py +++ b/lib/galaxy/tools/verify/test_data.py @@ -23,7 +23,7 @@ UPDATE_FAILED_TEMPLATE = Template( ) -LIST_SEP = re.compile("\s*,\s*") +LIST_SEP = re.compile(r"\s*,\s*") class TestDataResolver(object): diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 7b188671ffc..f1bfbe75bd0 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -640,9 +640,9 @@ def ready_name_for_url(raw_name): """ # Replace whitespace with '-' - slug_base = re.sub("\s+", "-", raw_name) + slug_base = re.sub(r"\s+", "-", raw_name) # Remove all non-alphanumeric characters. - slug_base = re.sub("[^a-zA-Z0-9\-]", "", slug_base) + slug_base = re.sub(r"[^a-zA-Z0-9\-]", "", slug_base) # Remove trailing '-'. if slug_base.endswith('-'): slug_base = slug_base[:-1] @@ -957,7 +957,7 @@ def listify(item, do_strip=False): def commaify(amount): orig = amount - new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', amount) + new = re.sub(r"^(-?\d+)(\d{3})", r'\g<1>,\g<2>', amount) if orig == new: return new else: @@ -1256,7 +1256,7 @@ def umask_fix_perms(path, umask, unmasked_perms, gid=None): perms = unmasked_perms & ~umask try: st = os.stat(path) - except OSError as e: + except OSError: log.exception('Unable to set permissions or group on %s', path) return # fix modes @@ -1364,8 +1364,8 @@ def size_to_bytes(size): except ValueError: pass # Otherwise it must have non-numeric characters - size_re = re.compile('([\d\.]+)\s*([eptgmk]b?|b|bytes?)$') - size_match = re.match(size_re, size.lower()) + size_re = re.compile(r'([\d\.]+)\s*([eptgmk]b?|b|bytes?)$') + size_match = size_re.match(size.lower()) assert size_match is not None size = float(size_match.group(1)) multiple = size_match.group(2) @@ -1557,7 +1557,7 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False) def parse_non_hex_float(s): - """ + r""" Parse string `s` into a float but throw a `ValueError` if the string is in the otherwise acceptable format `\d+e\d+` (e.g. 40000000000000e5.) diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index 776d0a0eeb1..321bcda8313 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -30,11 +30,11 @@ def check_html(file_path, chunk=None): temp = chunk.splitlines() else: temp = chunk - regexp1 = re.compile("]*HREF[^>]+>", re.I) - regexp2 = re.compile("]*>", re.I) - regexp3 = re.compile("]*>", re.I) - regexp4 = re.compile("]*>", re.I) - regexp5 = re.compile("]*>", re.I) + regexp1 = re.compile(r"]*HREF[^>]+>", re.I) + regexp2 = re.compile(r"]*>", re.I) + regexp3 = re.compile(r"]*>", re.I) + regexp4 = re.compile(r"]*>", re.I) + regexp5 = re.compile(r"]*>", re.I) lineno = 0 # TODO: Potentially reading huge lines into string here, this should be # reworked. diff --git a/lib/galaxy/util/inflection.py b/lib/galaxy/util/inflection.py index 059d1e6fe0e..a0ed931d083 100644 --- a/lib/galaxy/util/inflection.py +++ b/lib/galaxy/util/inflection.py @@ -45,9 +45,9 @@ class Base(object): "underscored_word". This can be really useful for creating friendly URLs.''' - return re.sub('[^A-Z^a-z^0-9^\/]+', '_', - re.sub('([a-z\d])([A-Z])', '\\1_\\2', - re.sub('([A-Z]+)([A-Z][a-z])', '\\1_\\2', re.sub('::', '/', word)))).lower() + return re.sub(r'[^A-Z^a-z^0-9^\/]+', r'_', + re.sub(r'([a-z\d])([A-Z])', r'\1_\2', + re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', re.sub(r'::', r'/', word)))).lower() def humanize(self, word, uppercase=''): '''Returns a human-readable string from word diff --git a/lib/galaxy/util/json.py b/lib/galaxy/util/json.py index 178ae393d32..c0fc6d54115 100644 --- a/lib/galaxy/util/json.py +++ b/lib/galaxy/util/json.py @@ -109,7 +109,7 @@ def validate_jsonrpc_request(request, regular_methods, notification_methods): data=str(e))) try: assert request['method'] in (regular_methods + notification_methods) - except AssertionError as e: + except AssertionError: return False, request, jsonrpc_response(request=request, error=dict(code=-32601, message='Method not found', @@ -142,14 +142,14 @@ def validate_jsonrpc_response(response, id=None): 'The "code" member of the "error" object in the Response is missing or not an integer.' assert 'message' in response, \ 'The "message" member of the "error" object in the Response is missing.' - except Exception as e: - log.error('Response was not valid JSON-RPC: %s' % str(e)) + except Exception: + log.exception('Response was not valid JSON-RPC') log.debug('Response was: %s' % response) return False, response if id is not None: try: assert 'id' in response and response['id'] == id - except Exception as e: + except Exception: log.error('The response id "%s" does not match the request id "%s"' % (response['id'], id)) return False, response return True, response diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index e6b0a650698..1c964618912 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -653,7 +653,7 @@ class VcfDataProvider(GenomeDataProvider): return ref_in_alt_index, alt[ref_in_alt_index + 1:], [[cig_ops.find("I"), alt_len - ref_len]] # Pack data. - genotype_re = re.compile('/|\|') + genotype_re = re.compile(r'/|\|') for count, line in enumerate(iterator): if count < start_val: continue diff --git a/lib/galaxy/visualization/data_providers/phyloviz/newickparser.py b/lib/galaxy/visualization/data_providers/phyloviz/newickparser.py index 2066cacd730..da398bf48f9 100644 --- a/lib/galaxy/visualization/data_providers/phyloviz/newickparser.py +++ b/lib/galaxy/visualization/data_providers/phyloviz/newickparser.py @@ -36,8 +36,8 @@ class Newick_Parser(Base_Parser): return self.phyloTree.generateJsonableDict() def cleanNewickString(self, rawNewick): - """removing semi colon, and illegal json characters (\,',") and white spaces""" - return re.sub(r'\s|;|\"|\'|\\', '', rawNewick) + r"""removing semi colon, and illegal json characters (\,',") and white spaces""" + return re.sub(r'\s|;|\"|\'|\\', r'', rawNewick) def _makeNodesFromString(self, string, depth): """elements separated by comma could be empty""" diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index e3b135e55a5..c222ff24741 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -57,7 +57,7 @@ SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error" def _is_valid_slug(slug): """ Returns true if slug is valid. """ - VALID_SLUG_RE = re.compile("^[a-z0-9\-]+$") + VALID_SLUG_RE = re.compile(r"^[a-z0-9\-]+$") return VALID_SLUG_RE.match(slug) diff --git a/lib/galaxy/web/framework/middleware/batch.py b/lib/galaxy/web/framework/middleware/batch.py index 7ad23f2d2e0..bd169c352cb 100644 --- a/lib/galaxy/web/framework/middleware/batch.py +++ b/lib/galaxy/web/framework/middleware/batch.py @@ -58,9 +58,9 @@ class BatchMiddleware(object): DEFAULT_CONFIG = { 'route' : '/api/batch', 'allowed_routes' : [ - '^api\/users.*', - '^api\/histories.*', - '^api\/jobs.*', + r'^api\/users.*', + r'^api\/histories.*', + r'^api\/jobs.*', ] } diff --git a/lib/galaxy/webapps/galaxy/api/uploads.py b/lib/galaxy/webapps/galaxy/api/uploads.py index c87c0c76351..6564d065e03 100644 --- a/lib/galaxy/webapps/galaxy/api/uploads.py +++ b/lib/galaxy/webapps/galaxy/api/uploads.py @@ -5,7 +5,7 @@ import logging import os import re -from galaxy.exceptions import MessageException, NotImplemented +from galaxy import exceptions from galaxy.web import expose_api_anonymous from galaxy.web.base.controller import BaseAPIController @@ -18,7 +18,7 @@ class UploadsAPIController(BaseAPIController): @expose_api_anonymous def index(self, trans, **kwd): - raise NotImplemented("Listing uploads is not implemented.") + raise exceptions.NotImplemented("Listing uploads is not implemented.") @expose_api_anonymous def create(self, trans, payload, **kwd): @@ -28,21 +28,21 @@ class UploadsAPIController(BaseAPIController): session_id = payload.get("session_id") session_start = payload.get("session_start") session_chunk = payload.get("session_chunk") - if re.match('^[\w-]+$', session_id) is None: - raise MessageException("Requires a session id.") + if re.match(r'^[\w-]+$', session_id) is None: + raise exceptions.MessageException("Requires a session id.") if session_start is None: - raise MessageException("Requires a session start.") + raise exceptions.MessageException("Requires a session start.") if not hasattr(session_chunk, "file"): - raise MessageException("Requires a session chunk.") + raise exceptions.MessageException("Requires a session chunk.") target_file = os.path.join(trans.app.config.new_file_path, session_id) target_size = 0 if os.path.exists(target_file): target_size = os.path.getsize(target_file) if session_start != target_size: - raise MessageException("Incorrect session start.") + raise exceptions.MessageException("Incorrect session start.") chunk_size = os.fstat(session_chunk.file.fileno()).st_size if chunk_size > trans.app.config.chunk_upload_size: - raise MessageException("Invalid chunk size.") + raise exceptions.MessageException("Invalid chunk size.") with open(target_file, "ab") as f: while True: read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 34edc91a572..d0c401f7aa0 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -568,13 +568,13 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cre ''' Validate email and username using regex ''' if email == '' or not isinstance(email, six.string_types): return 'Please provide your email address.' - if not re.match('^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$', email): + if not re.match(r'^(([^<>()[\]\.,;:\s@"]+(\.[^<>()[\]\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$', email): return 'Please provide your valid email address.' if len(email) > 255: return 'Email cannot be more than 255 characters in length.' def _validate_publicname(self, username): - if not re.match('^[a-z0-9\-]{3,255}$', username): + if not re.match(r'^[a-z0-9\-]{3,255}$', username): return 'Public name must contain only lowercase letters, numbers and "-". It also has to be shorter than 255 characters but longer than 2.' @expose_api diff --git a/lib/galaxy/webapps/galaxy/controllers/forms.py b/lib/galaxy/webapps/galaxy/controllers/forms.py index bea7e5311b9..ae468d89602 100644 --- a/lib/galaxy/webapps/galaxy/controllers/forms.py +++ b/lib/galaxy/webapps/galaxy/controllers/forms.py @@ -11,7 +11,7 @@ from galaxy.web.framework.helpers import grids, iff, time_ago log = logging.getLogger(__name__) -VALID_FIELDNAME_RE = re.compile("^[a-zA-Z0-9\_]+$") +VALID_FIELDNAME_RE = re.compile(r"^[a-zA-Z0-9\_]+$") class FormsGrid(grids.Grid): diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 0b9135f675b..e30307758f6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -305,7 +305,7 @@ class _PageContentProcessor(HTMLParser, object): Processes page content to produce HTML that is suitable for display. For now, processor renders embedded objects. """ - bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") + bare_ampersand = re.compile(r"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") elements_no_end_tag = set([ 'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame', 'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param', diff --git a/lib/galaxy/webapps/tool_shed/api/tools.py b/lib/galaxy/webapps/tool_shed/api/tools.py index e22274b0ceb..b72bd499edb 100644 --- a/lib/galaxy/webapps/tool_shed/api/tools.py +++ b/lib/galaxy/webapps/tool_shed/api/tools.py @@ -3,14 +3,10 @@ import logging from collections import namedtuple from galaxy import ( + exceptions, util, web ) -from galaxy.exceptions import ( - ConfigDoesNotAllowException, - NotImplemented, - RequestParameterInvalidException -) from galaxy.tools.parameters import params_to_strings from galaxy.tools.repositories import ValidationContext from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless @@ -62,7 +58,7 @@ class ToolsController(BaseAPIController): """ q = kwd.get('q', '') if not q: - raise NotImplemented('Listing of all the tools is not implemented. Provide parameter "q" to search instead.') + raise exceptions.NotImplemented('Listing of all the tools is not implemented. Provide parameter "q" to search instead.') else: page = kwd.get('page', 1) page_size = kwd.get('page_size', 10) @@ -70,7 +66,7 @@ class ToolsController(BaseAPIController): page = int(page) page_size = int(page_size) except ValueError: - raise RequestParameterInvalidException('The "page" and "page_size" have to be integers.') + raise exceptions.RequestParameterInvalidException('The "page" and "page_size" have to be integers.') return_jsonp = util.asbool(kwd.get('jsonp', False)) callback = kwd.get('callback', 'callback') search_results = self._search(trans, q, page, page_size) @@ -90,12 +86,12 @@ class ToolsController(BaseAPIController): """ conf = self.app.config if not conf.toolshed_search_on: - raise ConfigDoesNotAllowException('Searching the TS through the API is turned off for this instance.') + raise exceptions.ConfigDoesNotAllowException('Searching the TS through the API is turned off for this instance.') if not conf.whoosh_index_dir: - raise ConfigDoesNotAllowException('There is no directory for the search index specified. Please contact the administrator.') + raise exceptions.ConfigDoesNotAllowException('There is no directory for the search index specified. Please contact the administrator.') search_term = q.strip() if len(search_term) < 3: - raise RequestParameterInvalidException('The search term has to be at least 3 characters long.') + raise exceptions.RequestParameterInvalidException('The search term has to be at least 3 characters long.') tool_search = ToolSearch() diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index f6ac146a7a6..4baaaeb0798 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2380,7 +2380,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): if os.path.isdir(absolute_selected_file): try: os.rmdir(absolute_selected_file) - except OSError as e: + except OSError: # The directory is not empty pass elif os.path.isfile(absolute_selected_file): @@ -2388,7 +2388,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): dir = os.path.split(absolute_selected_file)[0] try: os.rmdir(dir) - except OSError as e: + except OSError: # The directory is not empty pass # Commit the change set. diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index e3e9cc26928..57a7f8a5f7c 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -116,7 +116,7 @@ class UploadController(BaseUIController): else: tar = tarfile.open(uploaded_file_name) istar = True - except tarfile.ReadError as e: + except tarfile.ReadError: tar = None istar = False else: diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 917b2ead48c..b83281196c9 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -768,7 +768,7 @@ class RegexReplace(RecipeStep): def execute_step(self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False): - """ + r""" Search and replace text in a file using regular expressions. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 2d95c0008e0..64277418093 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -192,7 +192,7 @@ def handle_directory_changes(app, host, username, repository, full_path, filenam if os.path.isdir(absolute_selected_file): try: os.rmdir(absolute_selected_file) - except OSError as e: + except OSError: # The directory is not empty. pass elif os.path.isfile(absolute_selected_file): @@ -200,7 +200,7 @@ def handle_directory_changes(app, host, username, repository, full_path, filenam dir = os.path.split(absolute_selected_file)[0] try: os.rmdir(dir) - except OSError as e: + except OSError: # The directory is not empty. pass # See if any admin users have chosen to receive email alerts when a repository is updated. diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 3aca9f02a20..4d92625bd0a 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -16,7 +16,7 @@ from tool_shed.util.web_util import escape log = logging.getLogger(__name__) -VALID_REPOSITORYNAME_RE = re.compile("^[a-z0-9\_]+$") +VALID_REPOSITORYNAME_RE = re.compile(r"^[a-z0-9\_]+$") def change_repository_name_in_hgrc_file(hgrc_file, new_name): @@ -813,10 +813,10 @@ def get_tool_shed_status_for_installed_repository(app, repository): # The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision. text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) return dict(revision_update=text) - except Exception as e: + except Exception: # The required tool shed may be unavailable, so default the revision_update value to 'false'. return dict(revision_update='false') - except Exception as e: + except Exception: log.exception("Error attempting to get tool shed status for installed repository %s", str(repository.name)) return {} diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index 8e4c22ee36f..194d6968b2c 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -626,7 +626,7 @@ def get_first_sentence(message): def process_sentence(message): # Strip tags like [15.07]. - message = re.sub("^\s*\[.*\]\s*", "", message) + message = re.sub(r"^\s*\[.*\]\s*", r"", message) # Link issues and pull requests... issue_url = "https://github.com/%s/%s/issues" % (PROJECT_OWNER, PROJECT_NAME) message = re.sub(r'#(\d+)', r'`#\1 <%s/\1>`__' % issue_url, message) diff --git a/scripts/check_python.py b/scripts/check_python.py index 91442e10d5e..7fb265ad2ff 100644 --- a/scripts/check_python.py +++ b/scripts/check_python.py @@ -39,5 +39,5 @@ def check_python(): if __name__ == '__main__': try: check_python() - except Exception as e: + except Exception: sys.exit(1) diff --git a/scripts/runtime_stats.py b/scripts/runtime_stats.py index 37e17468057..c9af08a2635 100755 --- a/scripts/runtime_stats.py +++ b/scripts/runtime_stats.py @@ -163,7 +163,7 @@ def query(tool_id=None, user=None, like=None, source='metrics', if like: query_tool_id = '%%/%s/%%' % tool_id - elif '/' in tool_id and not re.match('\d+\.\d+', tool_id.split('/')[-1]): + elif '/' in tool_id and not re.match(r'\d+\.\d+', tool_id.split('/')[-1]): query_tool_id = '%s%%' % tool_id like = True else: diff --git a/scripts/summarize_timings.py b/scripts/summarize_timings.py index 074c9bcdf4f..0db5a6c1a60 100644 --- a/scripts/summarize_timings.py +++ b/scripts/summarize_timings.py @@ -9,7 +9,7 @@ import numpy DESCRIPTION = "" -TIMING_LINE_PATTERN = re.compile("\((\d+.\d+) ms\)") +TIMING_LINE_PATTERN = re.compile(r"\((\d+.\d+) ms\)") def main(argv=None): diff --git a/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py b/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py index bfe3bac8d09..13cb8165710 100644 --- a/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py +++ b/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py @@ -20,8 +20,8 @@ from bootstrap_util import admin_user_info # noqa: I100,I201 log = logging.getLogger(__name__) -VALID_PUBLICNAME_RE = re.compile("^[a-z0-9\-]+$") -VALID_EMAIL_RE = re.compile("[^@]+@[^@]+\.[^@]+") +VALID_PUBLICNAME_RE = re.compile(r"^[a-z0-9\-]+$") +VALID_EMAIL_RE = re.compile(r"[^@]+@[^@]+\.[^@]+") class BootstrapApplication(object): diff --git a/scripts/transfer.py b/scripts/transfer.py index d04742641a5..82833dd510f 100644 --- a/scripts/transfer.py +++ b/scripts/transfer.py @@ -282,7 +282,7 @@ def scp_transfer(transfer_job): # TODO: add the ability to determine progress of the copy here like we do in the http_transfer above. cmd = "scp %s@%s:'%s' '%s'" % (user_name, host, - file_path.replace(' ', '\ '), + file_path.replace(' ', r'\ '), fn) pexpect.run(cmd, events={'.ssword:*': password + '\r\n', pexpect.TIMEOUT: print_ticks}, diff --git a/setup.cfg b/setup.cfg index 5fe373008ed..9f71b41c5cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ # E741 do not use variables named 'l', 'O' or 'I' # W503 is line breaks before binary operators, which has been reversed in PEP 8. # D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time). -ignore = E128,E203,E501,E402,E741,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413 +ignore = E128,E203,E501,E402,E741,W503,W504,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413 exclude = lib/galaxy/util/jstree.py # For flake8-import-order # https://github.com/PyCQA/flake8-import-order/blob/master/tests/test_cases/complete_smarkets.py diff --git a/test/functional/webhooks/phdcomics/__init__.py b/test/functional/webhooks/phdcomics/__init__.py index 56bf61f765a..8acdba77881 100644 --- a/test/functional/webhooks/phdcomics/__init__.py +++ b/test/functional/webhooks/phdcomics/__init__.py @@ -24,7 +24,7 @@ def main(trans, webhook, params): url = 'http://phdcomics.com/gradfeed.php' content = urlopen(url).read() soap = BeautifulSoup(content, 'html.parser') - pattern = '(?:http://www\.phdcomics\.com/comics\.php\?f=)(\d+)' + pattern = r'(?:http://www\.phdcomics\.com/comics\.php\?f=)(\d+)' webhook.config['latest_id'] = max([ int(re.search(pattern, link.text).group(1)) for link in soap.find_all('link', text=re.compile(pattern)) @@ -40,7 +40,7 @@ def main(trans, webhook, params): try: comic_src = comic_img[0].attrs.get('src') except IndexError: - pattern = '', u'']" diff --git a/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py b/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py index 1dca14476df..bee466b8796 100644 --- a/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py +++ b/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py @@ -324,7 +324,6 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_users' with 'lame_cluster'.") ) - # ================================Invalid yaml files============================== @log_capture() def test_no_file(self, l): diff --git a/test/unit/web/framework/test_webapp.py b/test/unit/web/framework/test_webapp.py index b14f4efd82e..98c20a54abb 100644 --- a/test/unit/web/framework/test_webapp.py +++ b/test/unit/web/framework/test_webapp.py @@ -56,7 +56,7 @@ class GalaxyWebTransaction_Headers_TestCase(unittest.TestCase): # should parse regex if using fwd slashes, string otherwise hostnames = config._parse_allowed_origin_hostnames({ - "allowed_origin_hostnames": "/host\d{2}/,geocities.com,miskatonic.edu" + "allowed_origin_hostnames": r"/host\d{2}/,geocities.com,miskatonic.edu" }) self.assertTrue(isinstance(hostnames[0], re._pattern_type)) self.assertTrue(isinstance(hostnames[1], str)) @@ -80,13 +80,13 @@ class GalaxyWebTransaction_Headers_TestCase(unittest.TestCase): self.assert_cors_header_equals(trans.response.headers, 'http://xxdarkhackerxx.disney.com') # subdomains should pass - trans = self._new_trans(allowed_origin_hostnames='something.com,/^[\w\.]*beep\.com/') + trans = self._new_trans(allowed_origin_hostnames=r'something.com,/^[\w\.]*beep\.com/') trans.request.headers['Origin'] = 'http://boop.beep.com' trans.set_cors_headers() self.assert_cors_header_equals(trans.response.headers, 'http://boop.beep.com') # ports should work - trans = self._new_trans(allowed_origin_hostnames='somethingelse.com,/^[\w\.]*beep\.com/') + trans = self._new_trans(allowed_origin_hostnames=r'somethingelse.com,/^[\w\.]*beep\.com/') trans.request.headers['Origin'] = 'http://boop.beep.com:8080' trans.set_cors_headers() self.assert_cors_header_equals(trans.response.headers, 'http://boop.beep.com:8080') @@ -104,7 +104,7 @@ class GalaxyWebTransaction_Headers_TestCase(unittest.TestCase): self.assert_cors_header_missing(trans.response.headers) # unicode should work - trans = self._new_trans(allowed_origin_hostnames='/öbb\.at/') + trans = self._new_trans(allowed_origin_hostnames=r'/öbb\.at/') trans.request.headers['Origin'] = 'http://öbb.at' trans.set_cors_headers() self.assertEqual( diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index d3dce6dfec2..91311533d5c 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -177,7 +177,7 @@ def add_composite_file(dataset, registry, output_path, files_path): if dataset.file_type is not None: try: datatype = registry.get_datatype_by_extension(dataset.file_type) - except Exception as e: + except Exception: print("Unable to instantiate the datatype object for the file type '%s'" % dataset.file_type) if dataset.composite_files: diff --git a/tools/extract/extract_genomic_dna.py b/tools/extract/extract_genomic_dna.py index 47279dd18ea..b42f7b127a8 100755 --- a/tools/extract/extract_genomic_dna.py +++ b/tools/extract/extract_genomic_dna.py @@ -212,7 +212,7 @@ def __main__(): nibs[chrom] = nib = bx.seq.nib.NibFile(open("%s/%s.nib" % (seq_path, chrom))) try: sequence = nib.get(start, end - start) - except Exception as e: + except Exception: warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " % (start, end - start, dbkey) warnings.append(warning) if not invalid_lines: diff --git a/tools/filters/convert_characters.py b/tools/filters/convert_characters.py index 7ff1cce07a1..15a1a9eb497 100644 --- a/tools/filters/convert_characters.py +++ b/tools/filters/convert_characters.py @@ -18,12 +18,12 @@ def __main__(): char_dict = { 'T': '\t', - 's': '\s', - 'Dt': '\.', + 's': r'\s', + 'Dt': r'\.', 'C': ',', 'D': '-', 'U': '_', - 'P': '\|', + 'P': r'\|', 'Co': ':', 'Sc': ';' } diff --git a/tools/filters/grep.py b/tools/filters/grep.py index d175009eb10..2cd8e0c4d3d 100644 --- a/tools/filters/grep.py +++ b/tools/filters/grep.py @@ -85,8 +85,8 @@ def main(): pattern = pattern.replace(value, key) # match filename and invert flag - fileRegEx = re.compile("^[A-Za-z0-9./\-_]+$") - invertRegEx = re.compile("(true)|(false)") + fileRegEx = re.compile(r"^[A-Za-z0-9./\-_]+$") + invertRegEx = re.compile(r"(true)|(false)") # verify that filename and inversion flag are in the correct format if not fileRegEx.match(outputfile): diff --git a/tools/filters/uniq.py b/tools/filters/uniq.py index 07bbefacb9b..9822aebcc5a 100644 --- a/tools/filters/uniq.py +++ b/tools/filters/uniq.py @@ -83,7 +83,7 @@ def main(): return -5 # All inputs have been specified at this point, now validate. - fileRegEx = re.compile("^[A-Za-z0-9./\-_]+$") + fileRegEx = re.compile(r"^[A-Za-z0-9./\-_]+$") columnRegEx = re.compile("([0-9]{1,},?)+") if not columnRegEx.match(columns): @@ -118,9 +118,9 @@ def main(): commandline += "-f " + columns # we want to remove *trailing* spaces from each field, # so look for spaces then tab (for first and middle selected columns) - # and replacw with just tab, and remove any spaces at end of the line + # and replace with just tab, and remove any spaces at end of the line # (for the final selected column): - commandline += " " + inputfile + " | sed 's/\ *\t/\t/' | sed 's/\ *$//'" + commandline += " " + inputfile + r" | sed 's/\ *\t/\t/' | sed 's/\ *$//'" commandline += " | sort | uniq -c" # uniq -C puts counts at the start, so we can sort lines by numerical value if sorting == "largest": @@ -130,7 +130,7 @@ def main(): # uniq -C produces lines with leading spaces, use sed to remove that # uniq -C puts a space between the count and the field, want a tab. # To replace just first tab, use sed again with 1 as the index - commandline += " | sed 's/^\ *//' | sed 's/ /\t/1' > " + outputfile + commandline += r" | sed 's/^\ *//' | sed 's/ /\t/1' > " + outputfile errorcode = subprocess.call(commandline, shell=True) print("Count of unique values in " + columns_for_display) diff --git a/tools/stats/filtering.py b/tools/stats/filtering.py index 6448302d80a..17047b6ec6e 100644 --- a/tools/stats/filtering.py +++ b/tools/stats/filtering.py @@ -185,7 +185,7 @@ if not check_expression(cond_text): # Work out which columns are used in the filter (save using 1 based counting) used_cols = sorted(set(int(match.group()[1:]) - for match in re.finditer('c(\d)+', cond_text))) + for match in re.finditer(r'c(\d)+', cond_text))) largest_col_index = max(used_cols) # Prepare the column variable names and wrappers for column data types. Only diff --git a/tools/stats/gsummary.py b/tools/stats/gsummary.py index 276a30f9349..b5dc153a949 100755 --- a/tools/stats/gsummary.py +++ b/tools/stats/gsummary.py @@ -42,7 +42,7 @@ def main(): if word and word not in math_allowed: stop_err("Invalid expression '%s': term '%s' is not recognized or allowed" % (expression, word)) symbols = set() - for symbol in re.compile('[^a-z0-9\s]+').findall(expression): + for symbol in re.compile(r'[^a-z0-9\s]+').findall(expression): if symbol and symbol not in ops_allowed: stop_err("Invalid expression '%s': operator '%s' is not recognized or allowed" % (expression, symbol)) else: