Merge pull request #5221 from nsoranzo/python3

Python3: fix moved imports

.ci/check_py3_compatibility.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+export ACK_OPTIONS=" --type python \
+--ignore-dir=.git/ \
+--ignore-dir=.tox/ \
+--ignore-dir=.venv/ \
+--ignore-dir=client/node_modules/ \
+--ignore-dir=database/ \
+--ignore-dir=doc/build/ \
+--ignore-dir=eggs/ \
+--ignore-dir=static/maps/ \
+--ignore-dir=static/scripts/"
+
+PYTHON2_ONLY_MODULES="__builtin__ _winreg BaseHTTPServer CGIHTTPServer \
+ConfigParser Cookie cookielib copy_reg cPickle cStringIO Dialog dummy_thread \
+FileDialog gdbm htmlentitydefs HTMLParser httplib Queue robotparser \
+ScrolledText SimpleDialog SimpleHTTPServer SimpleXMLRPCServer SocketServer \
+StringIO thread Tix tkColorChooser tkCommonDialog Tkconstants Tkdnd tkFont \
+Tkinter tkFileDialog tkMessageBox tkSimpleDialog ttk urllib urllib2 urlparse \
+xmlrpclib"
+
+ret=0
+for mod in $PYTHON2_ONLY_MODULES; do
+    ack "^import $mod(\n|\.)|^from $mod import "
+    if [ $? -eq 0 ]; then ret=1; fi
+done
+
+exit $ret

contrib/nagios/check_galaxy.py

@@ -13,10 +13,15 @@ import os
 import socket
 import sys
 import time
-import urllib2
 import warnings
 from user import home
 
+from six.moves.urllib.request import (
+    build_opener,
+    HTTPCookieProcessor,
+    Request
+)
+
 with warnings.catch_warnings():
     warnings.simplefilter('ignore')
     import twill.commands as tc
@@ -85,7 +90,7 @@ class Browser:
             dprint("no cookie jar at above path, creating")
             tc.save_cookies(self.cookie_jar)
         tc.load_cookies(self.cookie_jar)
-        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(tc.get_browser().cj))
+        self.opener = build_opener(HTTPCookieProcessor(tc.get_browser().cj))
 
     def get(self, path):
         tc.go("%s%s" % (self.server, path))
@@ -94,9 +99,9 @@ class Browser:
     def req(self, path, data=None, method=None):
         url = self.server + path
         if data:
-            req = urllib2.Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
+            req = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
         else:
-            req = urllib2.Request(url, headers={'Content-Type': 'application/json'})
+            req = Request(url, headers={'Content-Type': 'application/json'})
         if method:
             req.get_method = lambda: method
         res = self.opener.open(req)

lib/galaxy/jobs/runners/__init__.py

@@ -8,7 +8,8 @@ import string
 import subprocess
 import threading
 import time
-from Queue import (
+
+from six.moves.queue import (
     Empty,
     Queue
 )
@@ -439,7 +440,7 @@ class JobState(object):
                 job_name += '_%s' % self.job_wrapper.tool.old_id
             if self.job_wrapper.user:
                 job_name += '_%s' % self.job_wrapper.user
-            self.job_name = ''.join(map(lambda x: x if x in (string.ascii_letters + string.digits + '_') else '_', job_name))
+            self.job_name = ''.join(x if x in (string.ascii_letters + string.digits + '_') else '_' for x in job_name)
 
     @staticmethod
     def default_job_file(files_dir, id_tag):

lib/galaxy/util/biostar.py

@@ -6,10 +6,13 @@ from __future__ import absolute_import
 import hmac
 import logging
 import re
-import urlparse
 from unicodedata import normalize
 
 from six import text_type
+from six.moves.urllib.parse import (
+    urljoin,
+    urlsplit
+)
 
 from galaxy.tools.errors import ErrorReporter
 from galaxy.web.base.controller import url_for
@@ -71,7 +74,7 @@ def get_biostar_url(app, payload=None, biostar_action=None):
         payload[hmac_value_name] = smart_str(payload.get(hmac_value_name, ''), encoding='ascii', errors='replace')
         payload[hmac_parameter_name] = hmac.new(app.config.biostar_key, payload[hmac_value_name]).hexdigest()
     # generate url, can parse payload info
-    url = str(urlparse.urljoin(app.config.biostar_url, biostar_action.get('url')(payload)))
+    url = str(urljoin(app.config.biostar_url, biostar_action.get('url')(payload)))
     if not biostar_action.get('uses_payload'):
         payload = {}
     url = url_for(url)
@@ -136,8 +139,8 @@ def create_cookie(trans, key_name, key, email, age=DEFAULT_BIOSTAR_COOKIE_AGE, o
     value = "%s:%s" % (email, digest)
     trans.set_cookie(value, name=key_name, path='/', age=age, version='1')
     # We need to explicitly set the domain here, in order to allow for biostar in a subdomain to work
-    galaxy_hostname = urlparse.urlsplit(url_for('/', qualified=True)).hostname
-    biostar_hostname = urlparse.urlsplit(trans.app.config.biostar_url).hostname
+    galaxy_hostname = urlsplit(url_for('/', qualified=True)).hostname
+    biostar_hostname = urlsplit(trans.app.config.biostar_url).hostname
     trans.response.cookies[key_name]['domain'] = determine_cookie_domain(galaxy_hostname, biostar_hostname)
 
 

test/base/driver_util.py

@@ -1,7 +1,6 @@
 """Scripts for drivers of Galaxy functional tests."""
 
 import fcntl
-import httplib
 import json
 import logging
 import os
@@ -22,7 +21,10 @@ import nose.core
 import nose.loader
 import nose.plugins.manager
 from paste import httpserver
-from six.moves import shlex_quote
+from six.moves import (
+    http_client,
+    shlex_quote
+)
 from six.moves.urllib.parse import urlparse
 
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
@@ -400,7 +402,7 @@ def wait_for_http_server(host, port, sleep_amount=0.1, sleep_tries=150):
     # Test if the server is up
     for i in range(sleep_tries):
         # directly test the app, not the proxy
-        conn = httplib.HTTPConnection(host, port)
+        conn = http_client.HTTPConnection(host, port)
         try:
             conn.request("GET", "/")
             if conn.getresponse().status == 200:

test/functional/webhooks/phdcomics/helper/__init__.py

@@ -1,7 +1,8 @@
 import logging
 import random
 import re
-import urllib
+
+from six.moves.urllib.request import urlopen
 
 log = logging.getLogger(__name__)
 
@@ -21,7 +22,7 @@ def main(trans, webhook, params):
         # Get latest id
         if 'latest_id' not in webhook.config.keys():
             url = 'http://phdcomics.com/gradfeed.php'
-            content = urllib.urlopen(url).read()
+            content = urlopen(url).read()
             soap = BeautifulSoup(content, 'html.parser')
             pattern = '(?:http://www\.phdcomics\.com/comics\.php\?f=)(\d+)'
             webhook.config['latest_id'] = max([
@@ -32,7 +33,7 @@ def main(trans, webhook, params):
         random_id = random.randint(1, webhook.config['latest_id'])
         url = 'http://www.phdcomics.com/comics/archive.php?comicid=%d' % \
             random_id
-        content = urllib.urlopen(url).read()
+        content = urlopen(url).read()
         soup = BeautifulSoup(content, 'html.parser')
         comic_img = soup.find_all('img', id='comic2')