From 66944ba7de178cc45ffa7a4aa9253ec4512cb6fd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 16 Dec 2015 20:59:12 +0000 Subject: [PATCH] Lint with Python 3 serveral modules. Namely the modules I'd like to be shared with planemo, pulsar, etc.... --- .ci/flake8_py3_wrapper.sh | 5 + .ci/py3_sources.txt | 14 +++ lib/galaxy/jobs/metrics/collectl/processes.py | 4 + .../jobs/metrics/instrumenters/meminfo.py | 6 +- lib/galaxy/objectstore/s3_multipart_upload.py | 2 +- lib/galaxy/tools/toolbox/base.py | 6 +- lib/galaxy/tools/toolbox/integrated_panel.py | 2 +- lib/galaxy/util/__init__.py | 51 ++++----- .../util/backports/importlib/__init__.py | 2 + lib/galaxy/util/biostar.py | 7 +- lib/galaxy/util/dictobj.py | 5 + lib/galaxy/util/inflection.py | 2 +- lib/galaxy/util/json.py | 20 ++-- lib/galaxy/util/multi_byte.py | 4 +- lib/galaxy/util/object_wrapper.py | 10 +- lib/galaxy/util/pastescript/loadwsgi.py | 2 - lib/galaxy/util/pastescript/serve.py | 102 +++++++++--------- lib/galaxy/util/sanitize_html.py | 6 +- lib/galaxy/util/validation.py | 8 +- tox.ini | 8 +- 20 files changed, 163 insertions(+), 103 deletions(-) create mode 100755 .ci/flake8_py3_wrapper.sh create mode 100644 .ci/py3_sources.txt diff --git a/.ci/flake8_py3_wrapper.sh b/.ci/flake8_py3_wrapper.sh new file mode 100755 index 00000000000..64419fcb4e6 --- /dev/null +++ b/.ci/flake8_py3_wrapper.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` `paste -s .ci/py3_sources.txt` diff --git a/.ci/py3_sources.txt b/.ci/py3_sources.txt new file mode 100644 index 00000000000..1852fbd39b3 --- /dev/null +++ b/.ci/py3_sources.txt @@ -0,0 +1,14 @@ +lib/galaxy/util/ +lib/galaxy/jobs/runners/util/ +lib/pulsar/ +lib/galaxy/tools/parser/ +lib/galaxy/tools/lint.py +lib/galaxy/tools/lint_util.py +lib/galaxy/tools/loader.py +lib/galaxy/tools/loader_directory.py +lib/galaxy/tools/linters/ +lib/galaxy/tools/deps/ +lib/galaxy/tools/toolbox/ +lib/galaxy/tools/parser/ +lib/galaxy/jobs/metrics/ +lib/galaxy/objectstore/ diff --git a/lib/galaxy/jobs/metrics/collectl/processes.py b/lib/galaxy/jobs/metrics/collectl/processes.py index 3e7da0e20f0..ba144445d7b 100644 --- a/lib/galaxy/jobs/metrics/collectl/processes.py +++ b/lib/galaxy/jobs/metrics/collectl/processes.py @@ -3,12 +3,16 @@ statistics for a given pid's process and process ancestors. """ import collections import csv +import sys import tempfile from galaxy import util from ..collectl import stats +if sys.version_info > (3,): + long = int + import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/jobs/metrics/instrumenters/meminfo.py b/lib/galaxy/jobs/metrics/instrumenters/meminfo.py index ea10c1e7e55..7a9f3371e26 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/meminfo.py +++ b/lib/galaxy/jobs/metrics/instrumenters/meminfo.py @@ -1,10 +1,14 @@ import re - +import sys from galaxy import util from ..instrumenters import InstrumentPlugin from ...metrics import formatting +if sys.version_info > (3,): + long = int + + MEMINFO_LINE = re.compile(r"(\w+)\s*\:\s*(\d+) kB") # Important (non-verbose) meminfo property titles. diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py index ac922e3447f..9b40b70cbb4 100644 --- a/lib/galaxy/objectstore/s3_multipart_upload.py +++ b/lib/galaxy/objectstore/s3_multipart_upload.py @@ -24,7 +24,7 @@ except ImportError: def map_wrap(f): @functools.wraps(f) def wrapper(*args, **kwargs): - return apply(f, *args, **kwargs) + return f(*args, **kwargs) return wrapper diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index c2c68a2d56e..8831e7f7309 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -798,7 +798,7 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): tool = self._tools_by_id[ tool_id ] tarball_files = [] temp_files = [] - tool_xml = file( os.path.abspath( tool.config_file ), 'r' ).read() + tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read() # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path # modified to be relative to the repository root. image_found = False @@ -820,7 +820,7 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): if image_found: fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) os.close( fd ) - file( new_tool_config, 'w' ).write( tool_xml ) + open( new_tool_config, 'w' ).write( tool_xml ) tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) temp_files.append( os.path.abspath( new_tool_config ) ) else: @@ -880,7 +880,7 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): table_definition = table_definition % '\n'.join( data_table_definitions ) fd, table_conf = tempfile.mkstemp() os.close( fd ) - file( table_conf, 'w' ).write( table_definition ) + open( table_conf, 'w' ).write( table_definition ) tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) temp_files.append( table_conf ) # Create the tarball. diff --git a/lib/galaxy/tools/toolbox/integrated_panel.py b/lib/galaxy/tools/toolbox/integrated_panel.py index 6ebe5425963..c17d9c8387b 100644 --- a/lib/galaxy/tools/toolbox/integrated_panel.py +++ b/lib/galaxy/tools/toolbox/integrated_panel.py @@ -101,4 +101,4 @@ class ManagesIntegratedToolPanelMixin: shutil.copy( filename, filename + ".copy" ) filename = filename + ".copy" shutil.move( filename, destination ) - os.chmod( self._integrated_tool_panel_config, 0644 ) + os.chmod( self._integrated_tool_panel_config, 0o644 ) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 0835300c1ac..f78a565dca8 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -27,6 +27,9 @@ import urlparse from galaxy.util import json from datetime import datetime +from six import string_types, text_type +from six.moves import xrange + from email.MIMEText import MIMEText from os.path import relpath @@ -120,7 +123,7 @@ def file_iter(fname, sep=None): >>> len(lines) != 0 True """ - for line in file(fname): + for line in open(fname): if line and line[0] != '#': yield line.split(sep) @@ -169,7 +172,7 @@ def xml_to_string( elem, pretty=False ): elem = pretty_print_xml( elem ) try: return ElementTree.tostring( elem ) - except TypeError, e: + except TypeError as e: # we assume this is a comment if hasattr( elem, 'text' ): return "\n" % ( elem.text ) @@ -329,7 +332,7 @@ def pretty_print_time_interval( time=False, precise=False ): diff = now - datetime.fromtimestamp( time ) elif isinstance( time, datetime ): diff = now - time - elif isinstance( time, basestring ): + elif isinstance( time, string_types ): time = datetime.strptime( time, "%Y-%m-%dT%H:%M:%S.%f" ) diff = now - time else: @@ -417,7 +420,7 @@ def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_char """ if isinstance( text, list ): return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) - if not isinstance( text, basestring ): + if not isinstance( text, string_types ): text = smart_str( text ) return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) @@ -452,7 +455,7 @@ def sanitize_lists_to_string( values, valid_characters=valid_chars, character_ma def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" - if isinstance( value, basestring ): + if isinstance( value, string_types ): return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) @@ -712,7 +715,7 @@ falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) def asbool(obj): - if isinstance(obj, basestring): + if isinstance(obj, string_types): obj = obj.strip().lower() if obj in truthy: return True @@ -758,7 +761,7 @@ def listify( item, do_strip=False ): return [] elif isinstance( item, list ): return item - elif isinstance( item, basestring ) and item.count( ',' ): + elif isinstance( item, string_types ) and item.count( ',' ): if do_strip: return [token.strip() for token in item.split( ',' )] else: @@ -791,10 +794,10 @@ def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ) Returns a unicode string or None """ - if isinstance( value, unicode ): + if isinstance( value, text_type ): return value try: - return unicode( str( value ), encoding, error ) + return text_type( str( value ), encoding, error ) except: return default @@ -809,12 +812,12 @@ def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): """ if strings_only and isinstance(s, (type(None), int)): return s - if not isinstance(s, basestring): + if not isinstance(s, string_types): try: return str(s) except UnicodeEncodeError: - return unicode(s).encode(encoding, errors) - elif isinstance(s, unicode): + return text_type(s).encode(encoding, errors) + elif isinstance(s, text_type): return s.encode(encoding, errors) elif s and encoding != 'utf-8': return s.decode('utf-8', errors).encode(encoding, errors) @@ -931,7 +934,7 @@ def read_dbnames(filename): man_builds.sort() man_builds = [(build, name) for name, build in man_builds] db_names = DBNames( db_names + man_builds ) - except Exception, e: + except Exception as e: log.error( "ERROR: Unable to read builds file: %s", e ) if len(db_names) < 1: db_names = DBNames( [( db_names.default_value, db_names.default_name )] ) @@ -1012,7 +1015,7 @@ def mkstemp_ln( src, prefix='mkstemp_ln_' ): try: os.link( src, file ) return (os.path.abspath(file)) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: continue # try again raise @@ -1026,14 +1029,14 @@ def umask_fix_perms( path, umask, unmasked_perms, gid=None ): perms = unmasked_perms & ~umask try: st = os.stat( path ) - except OSError, e: + except OSError as e: log.exception( 'Unable to set permissions or group on %s' % path ) return # fix modes if stat.S_IMODE( st.st_mode ) != perms: try: os.chmod( path, perms ) - except Exception, e: + except Exception as e: log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), path, oct( perms ), @@ -1043,7 +1046,7 @@ def umask_fix_perms( path, umask, unmasked_perms, gid=None ): if gid is not None and st.st_gid != gid: try: os.chown( path, -1, gid ) - except Exception, e: + except Exception as e: try: desired_group = grp.getgrgid( gid ) current_group = grp.getgrgid( st.st_gid ) @@ -1165,26 +1168,26 @@ def send_mail( frm, to, subject, body, config ): try: s.starttls() log.debug( 'Initiated SSL/TLS connection to SMTP server: %s' % config.smtp_server ) - except RuntimeError, e: + except RuntimeError as e: log.warning( 'SSL/TLS support is not available to your Python interpreter: %s' % e ) - except smtplib.SMTPHeloError, e: + except smtplib.SMTPHeloError as e: log.error( "The server didn't reply properly to the HELO greeting: %s" % e ) s.close() raise - except smtplib.SMTPException, e: + except smtplib.SMTPException as e: log.warning( 'The server does not support the STARTTLS extension: %s' % e ) if config.smtp_username and config.smtp_password: try: s.login( config.smtp_username, config.smtp_password ) - except smtplib.SMTPHeloError, e: + except smtplib.SMTPHeloError as e: log.error( "The server didn't reply properly to the HELO greeting: %s" % e ) s.close() raise - except smtplib.SMTPAuthenticationError, e: + except smtplib.SMTPAuthenticationError as e: log.error( "The server didn't accept the username/password combination: %s" % e ) s.close() raise - except smtplib.SMTPException, e: + except smtplib.SMTPException as e: log.error( "No suitable authentication method was found: %s" % e ) s.close() raise @@ -1195,7 +1198,7 @@ def send_mail( frm, to, subject, body, config ): def force_symlink( source, link_name ): try: os.symlink( source, link_name ) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: os.remove( link_name ) os.symlink( source, link_name ) diff --git a/lib/galaxy/util/backports/importlib/__init__.py b/lib/galaxy/util/backports/importlib/__init__.py index 37b15bec4f1..6403ccf9e0d 100644 --- a/lib/galaxy/util/backports/importlib/__init__.py +++ b/lib/galaxy/util/backports/importlib/__init__.py @@ -3,6 +3,8 @@ # code compatible with Python 2.3. import sys +from six.moves import xrange + def _resolve_name(name, package, level): """Return the absolute name of the module to be imported.""" diff --git a/lib/galaxy/util/biostar.py b/lib/galaxy/util/biostar.py index 56dec8e2863..1f1761430b1 100644 --- a/lib/galaxy/util/biostar.py +++ b/lib/galaxy/util/biostar.py @@ -9,6 +9,7 @@ import re from unicodedata import normalize from galaxy.web.base.controller import url_for from galaxy.tools.errors import ErrorReporter +from six import text_type from . import smart_str log = logging.getLogger( __name__ ) @@ -41,14 +42,14 @@ def biostar_enabled( app ): # Slugifying from Armin Ronacher (http://flask.pocoo.org/snippets/5/) def slugify(text, delim=u'-'): """Generates an slightly worse ASCII-only slug.""" - if not isinstance( text, unicode ): - text = unicode( text ) + if not isinstance( text, text_type ): + text = text_type( text ) result = [] for word in _punct_re.split(text.lower()): word = normalize('NFKD', word).encode('ascii', 'ignore') if word: result.append(word) - return unicode(delim.join(result)) + return text_type(delim.join(result)) def get_biostar_url( app, payload=None, biostar_action=None ): diff --git a/lib/galaxy/util/dictobj.py b/lib/galaxy/util/dictobj.py index e25469ae310..5044d880c83 100644 --- a/lib/galaxy/util/dictobj.py +++ b/lib/galaxy/util/dictobj.py @@ -1,6 +1,11 @@ import pickle +def cmp(x, y): + # Builtin in Python 2, but not Python 3. + return (x > y) - (x < y) + + class DictionaryObject(object): """ Copyright 2012 "Grim Apps" diff --git a/lib/galaxy/util/inflection.py b/lib/galaxy/util/inflection.py index e03894c549f..efadb23ecba 100644 --- a/lib/galaxy/util/inflection.py +++ b/lib/galaxy/util/inflection.py @@ -269,7 +269,7 @@ class Inflector: def __init__( self, Inflector=English ): assert callable(Inflector), "Inflector should be a callable obj" - self.Inflector = apply(Inflector) + self.Inflector = Inflector() def pluralize(self, word) : '''Pluralizes nouns.''' diff --git a/lib/galaxy/util/json.py b/lib/galaxy/util/json.py index b5ecb42501b..1f9b5b44c4e 100644 --- a/lib/galaxy/util/json.py +++ b/lib/galaxy/util/json.py @@ -10,6 +10,8 @@ import math import random import string +from six import text_type, string_types + dumps = json.dumps loads = json.loads @@ -24,7 +26,7 @@ def json_fix( val ): return [ json_fix( v ) for v in val ] elif isinstance( val, dict ): return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] ) - elif isinstance( val, unicode ): + elif isinstance( val, text_type ): return val.encode( "utf8" ) else: return val @@ -34,7 +36,7 @@ def swap_inf_nan( val ): """ This takes an arbitrary object and preps it for jsonifying safely, templating Inf/NaN. """ - if isinstance(val, basestring): + if isinstance(val, string_types): # basestring first, because it's a sequence and would otherwise get caught below. return val elif isinstance( val, collections.Sequence ): @@ -76,7 +78,7 @@ def safe_dumps( *args, **kwargs ): def validate_jsonrpc_request( request, regular_methods, notification_methods ): try: request = loads( request ) - except Exception, e: + except Exception as e: return False, request, jsonrpc_response( id=None, error=dict( code=-32700, message='Parse error', @@ -87,14 +89,14 @@ def validate_jsonrpc_request( request, regular_methods, notification_methods ): assert request['jsonrpc'] == '2.0', \ 'Requested JSON-RPC version "%s" != required version "2.0".' % request['jsonrpc'] assert 'method' in request, 'No "method" member was sent with the Request object' - except AssertionError, e: + except AssertionError as e: return False, request, jsonrpc_response( request=request, error=dict( code=-32600, message='Invalid Request', data=str( e ) ) ) try: assert request['method'] in ( regular_methods + notification_methods ) - except AssertionError, e: + except AssertionError as e: return False, request, jsonrpc_response( request=request, error=dict( code=-32601, message='Method not found', @@ -102,7 +104,7 @@ def validate_jsonrpc_request( request, regular_methods, notification_methods ): try: if request['method'] in regular_methods: assert 'id' in request, 'No "id" member was sent with the Request object and the requested method "%s" is not a notification method' % request['method'] - except AssertionError, e: + except AssertionError as e: return False, request, jsonrpc_response( request=request, error=dict( code=-32600, message='Invalid Request', @@ -113,7 +115,7 @@ def validate_jsonrpc_request( request, regular_methods, notification_methods ): def validate_jsonrpc_response( response, id=None ): try: response = loads( response ) - except Exception, e: + except Exception as e: log.error( 'Response was not valid JSON: %s' % str( e ) ) log.debug( 'Response was: %s' % response ) return False, response @@ -127,14 +129,14 @@ def validate_jsonrpc_response( response, id=None ): 'The "code" member of the "error" object in the Response is missing or not an integer.' assert 'message' in response, \ 'The "message" member of the "error" object in the Response is missing.' - except Exception, e: + except Exception as e: log.error( 'Response was not valid JSON-RPC: %s' % str( e ) ) log.debug( 'Response was: %s' % response ) return False, response if id is not None: try: assert 'id' in response and response['id'] == id - except Exception, e: + except Exception as e: log.error( 'The response id "%s" does not match the request id "%s"' % ( response['id'], id ) ) return False, response return True, response diff --git a/lib/galaxy/util/multi_byte.py b/lib/galaxy/util/multi_byte.py index 133828c2fa7..b556c1b1396 100644 --- a/lib/galaxy/util/multi_byte.py +++ b/lib/galaxy/util/multi_byte.py @@ -1,10 +1,12 @@ import wchartype +from six import text_type + def is_multi_byte( chars ): for char in chars: try: - char = unicode( char ) + char = text_type( char ) except UnicodeDecodeError: # Probably binary return False diff --git a/lib/galaxy/util/object_wrapper.py b/lib/galaxy/util/object_wrapper.py index cc0ad04ce73..233171e6647 100644 --- a/lib/galaxy/util/object_wrapper.py +++ b/lib/galaxy/util/object_wrapper.py @@ -11,7 +11,9 @@ from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, Me BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) from UserDict import UserDict +import sys +from .dictobj import cmp from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string log = logging.getLogger( __name__ ) @@ -57,6 +59,10 @@ CHARACTER_MAP = { '>': '__gt__', INVALID_CHARACTER = "X" +if sys.version_info > (3, 0): + # __coerce__ doesn't do anything under Python anyway. + coerce = lambda x, y: x + def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) @@ -110,7 +116,7 @@ def wrap_with_safe_string( value, no_wrap_classes=None ): else: try: wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) - except TypeError, e: + except TypeError as e: # Fail-safe for when a class cannot be dynamically subclassed. log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) @@ -162,7 +168,7 @@ class SafeStringWrapper( object ): # that will be used when other + this (this + other is handled by __add__) try: return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) - except Exception, e: + except Exception as e: log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) return super( SafeStringWrapper, cls ).__new__( cls ) diff --git a/lib/galaxy/util/pastescript/loadwsgi.py b/lib/galaxy/util/pastescript/loadwsgi.py index 28b60508906..82ed7058eb5 100644 --- a/lib/galaxy/util/pastescript/loadwsgi.py +++ b/lib/galaxy/util/pastescript/loadwsgi.py @@ -28,7 +28,6 @@ def print_(template, *args, **kwargs): sys.stdout.writelines(template) if sys.version_info < (3, 0): - basestring = basestring from ConfigParser import ConfigParser from urllib import unquote iteritems = lambda d: d.iteritems() @@ -37,7 +36,6 @@ if sys.version_info < (3, 0): def reraise(t, e, tb): exec('raise t, e, tb', dict(t=t, e=e, tb=tb)) else: - basestring = str from configparser import ConfigParser from urllib.parse import unquote iteritems = lambda d: d.items() diff --git a/lib/galaxy/util/pastescript/serve.py b/lib/galaxy/util/pastescript/serve.py index be5c43dcbf3..ba924f8428a 100644 --- a/lib/galaxy/util/pastescript/serve.py +++ b/lib/galaxy/util/pastescript/serve.py @@ -16,6 +16,8 @@ # code here, stripping out uneeded functionality. # All top level imports from each package moved here and organized +from __future__ import print_function + import ConfigParser import atexit import errno @@ -359,20 +361,20 @@ class Command(object): class NotFoundCommand(Command): def run(self, args): - print ('Command %r not known (you may need to run setup.py egg_info)' - % self.command_name) + print('Command %r not known (you may need to run setup.py egg_info)' + % self.command_name) commands = list() commands.sort() if not commands: - print 'No commands registered.' - print 'Have you installed Paste Script?' - print '(try running python setup.py develop)' + print('No commands registered.') + print('Have you installed Paste Script?') + print('(try running python setup.py develop)') return 2 - print 'Known commands:' + print('Known commands:') longest = max([len(n) for n, c in commands]) for name, command in commands: - print ' %s %s' % (self.pad(name, length=longest), - command.load().summary) + print(' %s %s' % (self.pad(name, length=longest), + command.load().summary)) return 2 @@ -536,7 +538,7 @@ class ServeCommand(Command): if os.environ.get(self._reloader_environ_key): from paste import reloader if self.verbose > 1: - print 'Running reloading file monitor' + print('Running reloading file monitor') reloader.install(int(self.options.reload_interval)) if self.requires_config_file: reloader.watch_file(self.args[0]) @@ -553,7 +555,7 @@ class ServeCommand(Command): if cmd == 'restart' or cmd == 'stop': result = self.stop_daemon() if result: - print "Could not stop daemon" + print("Could not stop daemon") # It's ok to continue trying to restart if stop_daemon returns # a 1, otherwise shortcut and return. if cmd == 'restart' and result != 1: @@ -588,7 +590,7 @@ class ServeCommand(Command): if self.options.log_file: try: writeable_log_file = open(self.options.log_file, 'a') - except IOError, ioe: + except IOError as ioe: msg = 'Error: Unable to write to log file: %s' % ioe raise BadCommand(msg) writeable_log_file.close() @@ -597,7 +599,7 @@ class ServeCommand(Command): if self.options.pid_file: try: writeable_pid_file = open(self.options.pid_file, 'a') - except IOError, ioe: + except IOError as ioe: msg = 'Error: Unable to write to pid file: %s' % ioe raise BadCommand(msg) writeable_pid_file.close() @@ -605,9 +607,9 @@ class ServeCommand(Command): if getattr(self.options, 'daemon', False): try: self.daemonize() - except DaemonizeException, ex: + except DaemonizeException as ex: if self.verbose > 0: - print str(ex) + print(str(ex)) return if (self.options.monitor_restart and not @@ -641,20 +643,20 @@ class ServeCommand(Command): msg = 'Starting server in PID %i.' % os.getpid() else: msg = 'Starting server.' - print msg + print(msg) def serve(): try: server(app) - except (SystemExit, KeyboardInterrupt), e: + except (SystemExit, KeyboardInterrupt) as e: if self.verbose > 1: raise if str(e): msg = ' ' + str(e) else: msg = '' - print 'Exiting%s (-v to see traceback)' % msg - except AttributeError, e: + print('Exiting%s (-v to see traceback)' % msg) + except AttributeError as e: # Capturing bad error response from paste if str(e) == "'WSGIThreadPoolServer' object has no attribute 'thread_pool'": import socket @@ -665,7 +667,7 @@ class ServeCommand(Command): if jython_monitor: # JythonMonitor has to be ran from the main thread threading.Thread(target=serve).start() - print 'Starting Jython file monitor' + print('Starting Jython file monitor') jython_monitor.periodic_reload() else: serve() @@ -678,7 +680,7 @@ class ServeCommand(Command): % (pid, self.options.pid_file)) if self.verbose > 0: - print 'Entering daemon mode' + print('Entering daemon mode') pid = os.fork() if pid: # The forked process also has a handle on resources, so we @@ -717,7 +719,7 @@ class ServeCommand(Command): def record_pid(self, pid_file): pid = os.getpid() if self.verbose > 1: - print 'Writing PID %s to %s' % (pid, pid_file) + print('Writing PID %s to %s' % (pid, pid_file)) f = open(pid_file, 'w') f.write(str(pid)) f.close() @@ -726,19 +728,19 @@ class ServeCommand(Command): def stop_daemon(self): pid_file = self.options.pid_file or 'paster.pid' if not os.path.exists(pid_file): - print 'No PID file exists in %s' % pid_file + print('No PID file exists in %s' % pid_file) return 1 pid = read_pidfile(pid_file) if not pid: - print "Not a valid PID file in %s" % pid_file + print("Not a valid PID file in %s" % pid_file) return 1 pid = live_pidfile(pid_file) if not pid: - print "PID in %s is not valid (deleting)" % pid_file + print("PID in %s is not valid (deleting)" % pid_file) try: os.unlink(pid_file) - except (OSError, IOError), e: - print "Could not delete: %s" % e + except (OSError, IOError) as e: + print("Could not delete: %s" % e) return 2 return 1 for j in range(10): @@ -748,7 +750,7 @@ class ServeCommand(Command): os.kill(pid, signal.SIGTERM) time.sleep(1) else: - print "failed to kill web process %s" % pid + print("failed to kill web process %s" % pid) return 3 if os.path.exists(pid_file): os.unlink(pid_file) @@ -757,17 +759,17 @@ class ServeCommand(Command): def show_status(self): pid_file = self.options.pid_file or 'paster.pid' if not os.path.exists(pid_file): - print 'No PID file %s' % pid_file + print('No PID file %s' % pid_file) return 1 pid = read_pidfile(pid_file) if not pid: - print 'No PID in file %s' % pid_file + print('No PID in file %s' % pid_file) return 1 pid = live_pidfile(pid_file) if not pid: - print 'PID %s in %s is not running' % (pid, pid_file) + print('PID %s in %s is not running' % (pid, pid_file)) return 1 - print 'Server running in PID %s' % pid + print('Server running in PID %s' % pid) return 0 def restart_with_reloader(self): @@ -776,9 +778,9 @@ class ServeCommand(Command): def restart_with_monitor(self, reloader=False): if self.verbose > 0: if reloader: - print 'Starting subprocess with file monitor' + print('Starting subprocess with file monitor') else: - print 'Starting subprocess with monitor parent' + print('Starting subprocess with monitor parent') while 1: args = [self.quote_first_command_arg(sys.executable)] + sys.argv new_environ = os.environ.copy() @@ -794,7 +796,7 @@ class ServeCommand(Command): exit_code = proc.wait() proc = None except KeyboardInterrupt: - print '^C caught in monitor process' + print('^C caught in monitor process') if self.verbose > 1: raise return 1 @@ -813,7 +815,7 @@ class ServeCommand(Command): if exit_code != 3: return exit_code if self.verbose > 0: - print '-' * 20, 'Restarting', '-' * 20 + print('-' * 20, 'Restarting', '-' * 20) def change_user_group(self, user, group): if not user and not group: @@ -846,8 +848,8 @@ class ServeCommand(Command): gid = entry.pw_gid uid = entry.pw_uid if self.verbose > 0: - print 'Changing user to %s:%s (%s:%s)' % ( - user, group or '(unknown)', uid, gid) + print('Changing user to %s:%s (%s:%s)' % ( + user, group or '(unknown)', uid, gid)) if hasattr(os, 'initgroups'): os.initgroups(user, gid) else: @@ -907,7 +909,7 @@ def live_pidfile(pidfile): try: os.kill(int(pid), 0) return pid - except OSError, e: + except OSError as e: if e.errno == errno.EPERM: return pid return None @@ -943,26 +945,26 @@ def _remove_pid_file(written_pid, filename, verbosity): pass else: if pid_in_file != current_pid: - print "PID file %s contains %s, not expected PID %s" % ( - filename, pid_in_file, current_pid) + print("PID file %s contains %s, not expected PID %s" % ( + filename, pid_in_file, current_pid)) return if verbosity > 0: - print "Removing PID file %s" % filename + print("Removing PID file %s" % filename) try: os.unlink(filename) return - except OSError, e: + except OSError as e: # Record, but don't give traceback - print "Cannot remove PID file: %s" % e + print("Cannot remove PID file: %s" % e) # well, at least lets not leave the invalid PID around... try: f = open(filename, 'w') f.write('') f.close() - except OSError, e: - print 'Stale PID left in file: %s (%e)' % (filename, e) + except OSError as e: + print('Stale PID left in file: %s (%e)' % (filename, e)) else: - print 'Stale PID removed' + print('Stale PID removed') def ensure_port_cleanup(bound_addresses, maxtries=30, sleeptime=2): @@ -988,7 +990,7 @@ def _cleanup_ports(bound_addresses, maxtries=30, sleeptime=2): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect(bound_address) - except socket.error, e: + except socket.error as e: if e.args[0] != errno.ECONNREFUSED: raise break @@ -1049,7 +1051,7 @@ def run(args=None): if options.do_help: args = ['help'] + args if not args: - print 'Usage: %s COMMAND' % sys.argv[0] + print('Usage: %s COMMAND' % sys.argv[0]) args = ['help'] command_name = args[0] if command_name not in commands: @@ -1063,7 +1065,7 @@ def invoke(command, command_name, options, args): try: runner = command(command_name) exit_code = runner.run(args) - except BadCommand, e: - print e.message + except BadCommand as e: + print(e.message) exit_code = e.exit_code sys.exit(exit_code) diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 031eeef4b8a..5f34e9fdda1 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -5,6 +5,10 @@ HTML Sanitizer (ripped from feedparser) import re import sgmllib +from six import unichr +from six import text_type as unicode + + # reversable htmlentitydefs mappings for Python 2.2 try: from htmlentitydefs import name2codepoint, codepoint2name @@ -370,7 +374,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): # declare xlink namespace, if needed if self.mathmlOK or self.svgOK: - if filter(lambda (n, v): n.startswith('xlink:'), attrs): + if filter(lambda n, v: n.startswith('xlink:'), attrs): if not ('xmlns:xlink', 'http://www.w3.org/1999/xlink') in attrs: attrs.append(('xmlns:xlink', 'http://www.w3.org/1999/xlink')) diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py index 49832c97f1d..7b0686cec19 100644 --- a/lib/galaxy/util/validation.py +++ b/lib/galaxy/util/validation.py @@ -5,18 +5,20 @@ TODO: Refactor BaseController references to similar methods to use this module. from galaxy import exceptions from galaxy.util.sanitize_html import sanitize_html +from six import string_types, text_type + def validate_and_sanitize_basestring( key, val ): - if not isinstance( val, basestring ): + if not isinstance( val, string_types ): raise exceptions.RequestParameterInvalidException( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) - return unicode( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) + return text_type( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) def validate_and_sanitize_basestring_list( key, val ): try: assert isinstance( val, list ) - return [ unicode( sanitize_html( t, 'utf-8', 'text/html' ), 'utf-8' ) for t in val ] + return [ text_type( sanitize_html( t, 'utf-8', 'text/html' ), 'utf-8' ) for t in val ] except ( AssertionError, TypeError ): raise exceptions.RequestParameterInvalidException( '%s must be a list of strings: %s' % ( key, str( type( val ) ) ) ) diff --git a/tox.ini b/tox.ini index 7b102d7a642..f34ee71ab2c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27-lint, py26-lint, py27-unit, py26-unit, qunit, mako-count, web-controller-line-count +envlist = py27-lint, py26-lint, py27-unit, py26-unit, qunit, mako-count, web-controller-line-count, py34-lint skipsdist = True @@ -13,6 +13,12 @@ commands = bash .ci/flake8_wrapper.sh whitelist_externals = bash deps = flake8 +[testenv:py34-lint] +commands = bash .ci/flake8_py3_wrapper.sh +whitelist_externals = bash +deps = flake8 + + [testenv:py27-unit] commands = bash run_tests.sh --skip-venv -u whitelist_externals = bash