From 6f722861cfd4c36f93348048fe6e6c83305fcc9f Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 15 Dec 2017 01:02:40 +0000 Subject: [PATCH] Python3: fix moved imports Add `.ci/check_py3_compatibility.sh` script to find library modules that have been moved in Python3. Fix errors found by the script. --- .ci/check_py3_compatibility.sh | 28 +++++++++++++++++++ contrib/nagios/check_galaxy.py | 13 ++++++--- lib/galaxy/jobs/runners/__init__.py | 5 ++-- lib/galaxy/util/biostar.py | 11 +++++--- test/base/driver_util.py | 8 ++++-- .../webhooks/phdcomics/helper/__init__.py | 7 +++-- 6 files changed, 56 insertions(+), 16 deletions(-) create mode 100755 .ci/check_py3_compatibility.sh diff --git a/.ci/check_py3_compatibility.sh b/.ci/check_py3_compatibility.sh new file mode 100755 index 00000000000..a2310dac0a1 --- /dev/null +++ b/.ci/check_py3_compatibility.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +export ACK_OPTIONS=" --type python \ +--ignore-dir=.git/ \ +--ignore-dir=.tox/ \ +--ignore-dir=.venv/ \ +--ignore-dir=client/node_modules/ \ +--ignore-dir=database/ \ +--ignore-dir=doc/build/ \ +--ignore-dir=eggs/ \ +--ignore-dir=static/maps/ \ +--ignore-dir=static/scripts/" + +PYTHON2_ONLY_MODULES="__builtin__ _winreg BaseHTTPServer CGIHTTPServer \ +ConfigParser Cookie cookielib copy_reg cPickle cStringIO Dialog dummy_thread \ +FileDialog gdbm htmlentitydefs HTMLParser httplib Queue robotparser \ +ScrolledText SimpleDialog SimpleHTTPServer SimpleXMLRPCServer SocketServer \ +StringIO thread Tix tkColorChooser tkCommonDialog Tkconstants Tkdnd tkFont \ +Tkinter tkFileDialog tkMessageBox tkSimpleDialog ttk urllib urllib2 urlparse \ +xmlrpclib" + +ret=0 +for mod in $PYTHON2_ONLY_MODULES; do + ack "^import $mod(\n|\.)|^from $mod import " + if [ $? -eq 0 ]; then ret=1; fi +done + +exit $ret diff --git a/contrib/nagios/check_galaxy.py b/contrib/nagios/check_galaxy.py index 565d1d278be..57da403ec0f 100755 --- a/contrib/nagios/check_galaxy.py +++ b/contrib/nagios/check_galaxy.py @@ -13,10 +13,15 @@ import os import socket import sys import time -import urllib2 import warnings from user import home +from six.moves.urllib.request import ( + build_opener, + HTTPCookieProcessor, + Request +) + with warnings.catch_warnings(): warnings.simplefilter('ignore') import twill.commands as tc @@ -85,7 +90,7 @@ class Browser: dprint("no cookie jar at above path, creating") tc.save_cookies(self.cookie_jar) tc.load_cookies(self.cookie_jar) - self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(tc.get_browser().cj)) + self.opener = build_opener(HTTPCookieProcessor(tc.get_browser().cj)) def get(self, path): tc.go("%s%s" % (self.server, path)) @@ -94,9 +99,9 @@ class Browser: def req(self, path, data=None, method=None): url = self.server + path if data: - req = urllib2.Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data)) + req = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data)) else: - req = urllib2.Request(url, headers={'Content-Type': 'application/json'}) + req = Request(url, headers={'Content-Type': 'application/json'}) if method: req.get_method = lambda: method res = self.opener.open(req) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index ecf935bd0cb..b770c9b9f1b 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -8,7 +8,8 @@ import string import subprocess import threading import time -from Queue import ( + +from six.moves.queue import ( Empty, Queue ) @@ -439,7 +440,7 @@ class JobState(object): job_name += '_%s' % self.job_wrapper.tool.old_id if self.job_wrapper.user: job_name += '_%s' % self.job_wrapper.user - self.job_name = ''.join(map(lambda x: x if x in (string.ascii_letters + string.digits + '_') else '_', job_name)) + self.job_name = ''.join(x if x in (string.ascii_letters + string.digits + '_') else '_' for x in job_name) @staticmethod def default_job_file(files_dir, id_tag): diff --git a/lib/galaxy/util/biostar.py b/lib/galaxy/util/biostar.py index c0f8df585ca..81344b0b718 100644 --- a/lib/galaxy/util/biostar.py +++ b/lib/galaxy/util/biostar.py @@ -6,10 +6,13 @@ from __future__ import absolute_import import hmac import logging import re -import urlparse from unicodedata import normalize from six import text_type +from six.moves.urllib.parse import ( + urljoin, + urlsplit +) from galaxy.tools.errors import ErrorReporter from galaxy.web.base.controller import url_for @@ -71,7 +74,7 @@ def get_biostar_url(app, payload=None, biostar_action=None): payload[hmac_value_name] = smart_str(payload.get(hmac_value_name, ''), encoding='ascii', errors='replace') payload[hmac_parameter_name] = hmac.new(app.config.biostar_key, payload[hmac_value_name]).hexdigest() # generate url, can parse payload info - url = str(urlparse.urljoin(app.config.biostar_url, biostar_action.get('url')(payload))) + url = str(urljoin(app.config.biostar_url, biostar_action.get('url')(payload))) if not biostar_action.get('uses_payload'): payload = {} url = url_for(url) @@ -136,8 +139,8 @@ def create_cookie(trans, key_name, key, email, age=DEFAULT_BIOSTAR_COOKIE_AGE, o value = "%s:%s" % (email, digest) trans.set_cookie(value, name=key_name, path='/', age=age, version='1') # We need to explicitly set the domain here, in order to allow for biostar in a subdomain to work - galaxy_hostname = urlparse.urlsplit(url_for('/', qualified=True)).hostname - biostar_hostname = urlparse.urlsplit(trans.app.config.biostar_url).hostname + galaxy_hostname = urlsplit(url_for('/', qualified=True)).hostname + biostar_hostname = urlsplit(trans.app.config.biostar_url).hostname trans.response.cookies[key_name]['domain'] = determine_cookie_domain(galaxy_hostname, biostar_hostname) diff --git a/test/base/driver_util.py b/test/base/driver_util.py index b9c83a50d28..29e99a8aef7 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -1,7 +1,6 @@ """Scripts for drivers of Galaxy functional tests.""" import fcntl -import httplib import json import logging import os @@ -22,7 +21,10 @@ import nose.core import nose.loader import nose.plugins.manager from paste import httpserver -from six.moves import shlex_quote +from six.moves import ( + http_client, + shlex_quote +) from six.moves.urllib.parse import urlparse from galaxy.app import UniverseApplication as GalaxyUniverseApplication @@ -400,7 +402,7 @@ def wait_for_http_server(host, port, sleep_amount=0.1, sleep_tries=150): # Test if the server is up for i in range(sleep_tries): # directly test the app, not the proxy - conn = httplib.HTTPConnection(host, port) + conn = http_client.HTTPConnection(host, port) try: conn.request("GET", "/") if conn.getresponse().status == 200: diff --git a/test/functional/webhooks/phdcomics/helper/__init__.py b/test/functional/webhooks/phdcomics/helper/__init__.py index d3ec4d7fbe5..56bf61f765a 100644 --- a/test/functional/webhooks/phdcomics/helper/__init__.py +++ b/test/functional/webhooks/phdcomics/helper/__init__.py @@ -1,7 +1,8 @@ import logging import random import re -import urllib + +from six.moves.urllib.request import urlopen log = logging.getLogger(__name__) @@ -21,7 +22,7 @@ def main(trans, webhook, params): # Get latest id if 'latest_id' not in webhook.config.keys(): url = 'http://phdcomics.com/gradfeed.php' - content = urllib.urlopen(url).read() + content = urlopen(url).read() soap = BeautifulSoup(content, 'html.parser') pattern = '(?:http://www\.phdcomics\.com/comics\.php\?f=)(\d+)' webhook.config['latest_id'] = max([ @@ -32,7 +33,7 @@ def main(trans, webhook, params): random_id = random.randint(1, webhook.config['latest_id']) url = 'http://www.phdcomics.com/comics/archive.php?comicid=%d' % \ random_id - content = urllib.urlopen(url).read() + content = urlopen(url).read() soup = BeautifulSoup(content, 'html.parser') comic_img = soup.find_all('img', id='comic2')