diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index f6b2b8efc2f..4775897548f 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -17,8 +17,8 @@ import fileinput import os import sys +import requests from six.moves.urllib.parse import urlencode -from six.moves.urllib.request import urlopen import parse_builds @@ -36,8 +36,8 @@ def getchrominfo(url, db): "hgta_regionType": "", "position": "", "hgta_doTopSubmit": "get info"}) - page = urlopen(URL) - for line in page: + page = requests.get(URL).text + for line in page.split('\n'): line = line.rstrip("\r\n") if line.startswith("#"): continue diff --git a/cron/parse_builds.py b/cron/parse_builds.py index c30197184f8..243440a331b 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -9,18 +9,17 @@ from __future__ import print_function import sys import xml.etree.ElementTree as ElementTree -from six.moves.urllib.request import urlopen +import requests def getbuilds(url): try: - page = urlopen(url) + text = requests.get(url).text except: print("#Unable to open " + url) print("?\tunspecified (?)") sys.exit(1) - text = page.read() try: tree = ElementTree.fromstring(text) except: diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index b22cd449bf9..4ae43e52765 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -6,7 +6,7 @@ from __future__ import print_function import xml.etree.ElementTree as ElementTree -from six.moves.urllib.request import urlopen +import requests sites = ['http://genome.ucsc.edu/cgi-bin/', 'http://archaea.ucsc.edu/cgi-bin/', @@ -20,11 +20,11 @@ def main(): trackurl = sites[i] + "hgTracks?" builds = [] try: - page = urlopen(site) + text = requests.get(site).text except: print("#Unable to connect to " + site) continue - text = page.read() + try: tree = ElementTree.fromstring(text) except: diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 4b33dbe8d5a..6cebdd82e14 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -306,7 +306,7 @@ def parse_containers_config(containers_config_file): conf = DEFAULT_CONF.copy() try: with open(containers_config_file) as fh: - c = yaml.load(fh) + c = yaml.safe_load(fh) conf.update(c.get('containers', {})) except (OSError, IOError) as exc: if exc.errno == errno.ENOENT: diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b4ab524b778..87cf9aeee92 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -252,12 +252,8 @@ class Bam(Binary): message = 'Attempting to use functionality requiring samtools, but it cannot be located on Galaxy\'s PATH.' raise Exception(message) - # Get the version of samtools via --version-only, if available - p = subprocess.Popen(['samtools', '--version-only'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() - # --version-only is available # Format is +htslib- if p.returncode == 0: @@ -294,10 +290,8 @@ class Bam(Binary): def _is_coordinate_sorted(self, file_name): """See if the input BAM file is sorted from the header information.""" - params = ["samtools", "view", "-H", file_name] - output = subprocess.Popen(params, stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0] - # find returns -1 if string is not found - return output.find("SO:coordinate") != -1 or output.find("SO:sorted") != -1 + output = subprocess.check_output(["samtools", "view", "-H", file_name]) + return 'SO:coordinate' in output or 'SO:sorted' in output def dataset_content_needs_grooming(self, file_name): """See if file_name is a sorted BAM file""" @@ -322,8 +316,7 @@ class Bam(Binary): return False index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name stderr_name = tempfile.NamedTemporaryFile(prefix="bam_index_stderr").name - command = 'samtools index %s %s' % (file_name, index_name) - proc = subprocess.Popen(args=command, shell=True, stderr=open(stderr_name, 'wb')) + proc = subprocess.Popen(['samtools', 'index', file_name, index_name], stderr=open(stderr_name, 'wb')) proc.wait() stderr = open(stderr_name).read().strip() if stderr: @@ -366,8 +359,8 @@ class Bam(Binary): tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted') stderr_name = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix="bam_sort_stderr").name samtools_created_sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix # samtools accepts a prefix, not a filename, it always adds .bam to the prefix - command = "samtools sort %s %s" % (file_name, tmp_sorted_dataset_file_name_prefix) - proc = subprocess.Popen(args=command, shell=True, cwd=tmp_dir, stderr=open(stderr_name, 'wb')) + proc = subprocess.Popen(['samtools', 'sort', file_name, tmp_sorted_dataset_file_name_prefix], + cwd=tmp_dir, stderr=open(stderr_name, 'wb')) exit_code = proc.wait() # Did sort succeed? stderr = open(stderr_name).read().strip() @@ -1309,11 +1302,8 @@ class ExcelXls(Binary): edam_format = "format_3468" def sniff(self, filename): - mime_type = subprocess.check_output("file --mime-type '{}'".format(filename), shell=True).rstrip() - if mime_type.find("application/vnd.ms-excel") != -1: - return True - else: - return False + mime_type = subprocess.check_output(['file', '--mime-type', filename]) + return "application/vnd.ms-excel" in mime_type def get_mime(self): """Returns the mime type of the datatype""" diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index b0b8bb4695c..d4ddd98927b 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -133,8 +133,9 @@ if __name__ == "__main__": # Sort through a tempfile first temp_file = tempfile.NamedTemporaryFile(mode="r") environ['LC_ALL'] = 'POSIX' - commandline = "sort -f -n -k %d -k %d -k %d -o %s %s" % (chr_col_1 + 1, start_col_1 + 1, end_col_1 + 1, temp_file.name, in_fname) - subprocess.check_call(commandline, shell=True) + subprocess.check_call([ + 'sort', '-f', '-n', '-k', chr_col_1 + 1, '-k', start_col_1 + 1, '-k', end_col_1 + 1, '-o', temp_file.name, in_fname + ]) coverage = CoverageWriter(out_stream=open(out_fname, "a"), chromCol=chr_col_2, positionCol=position_col_2, diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py index 777259c3e0e..83b8050e46a 100644 --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -72,9 +72,9 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): if not missval: print('### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf) missval = '0' - cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink, inpedfilepath, outroot, missval) - p = subprocess.Popen(cl, shell=True, cwd=outfilepath) - p.wait() # run plink + subprocess.check_call([plink, '--noweb', '--file', inpedfilepath, + '--make-bed', '--out', outroot, + '--missing-genotype', missval], cwd=outfilepath) def main(): diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 86c4a4d051f..fbe1834b8b1 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -41,8 +41,7 @@ def pruneLD(plinktasks=[], cd='./', vclbase=[]): for task in plinktasks: # each is a list vcl = vclbase + task with open(plog, 'w') as sto: - x = subprocess.Popen(' '.join(vcl), shell=True, stdout=sto, stderr=sto, cwd=cd) - x.wait() + subprocess.check_call(vcl, stdout=sto, stderr=sto, cwd=cd) try: lplog = open(plog, 'r').readlines() lplog = [elem for elem in lplog if elem.find('Pruning SNP') == -1] diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py index 023c6ee5f30..24a052384a3 100644 --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -40,9 +40,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): """ basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath, basename) - cl = '%s --noweb --bfile %s --recode --out %s ' % (plink, inpedfilepath, outroot) - p = subprocess.Popen(cl, shell=True, cwd=outfilepath) - p.wait() # run plink + subprocess.check_call([plink, '--noweb', '--bfile', inpedfilepath, '--recode', '--out', outroot], cwd=outfilepath) def main(): diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index dcc84b6c518..8c54258eb9a 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -34,11 +34,8 @@ def _get_samtools_version(): if not cmd_exists('samtools'): raise Exception('This tool needs samtools, but it is not on PATH.') # Get the version of samtools via --version-only, if available - p = subprocess.Popen(['samtools', '--version-only'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() - # --version-only is available # Format is +htslib- if p.returncode == 0: diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 31902b14513..f11b8498719 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -359,7 +359,7 @@ class Registry(object): build_sites_config_file = getattr(self.config, "build_sites_config_file", None) if build_sites_config_file and os.path.exists(build_sites_config_file): with open(build_sites_config_file, "r") as f: - build_sites_config = yaml.load(f) + build_sites_config = yaml.safe_load(f) if not isinstance(build_sites_config, list): self.log.exception("Build sites configuration YAML file does not declare list of sites.") return diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 1267ff3da4a..ee47ad70bbd 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -7,6 +7,7 @@ import logging import os import re import string +import subprocess import sys from cgi import escape from itertools import islice @@ -693,8 +694,7 @@ class BaseFastq (Sequence): else: commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) for cmd in commands: - if 0 != os.system(cmd): - raise Exception("Executing '%s' failed" % cmd) + subprocess.check_call(cmd, shell=True) return True process_split_file = staticmethod(process_split_file) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 0100448e65a..234a436a648 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -521,15 +521,12 @@ class Sam(Tabular): Multiple SAM files may each have headers. Since the headers should all be the same, remove the headers from files 1-n, keeping them in the first file only """ - cmd = 'mv %s %s' % (split_files[0], output_file) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + shutil.move(split_files[0], output_file) + if len(split_files) > 1: - cmd = 'egrep -v -h "^@" %s >> %s' % (' '.join(split_files[1:]), output_file) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file] + subprocess.check_call(cmd, shell=True) + merge = staticmethod(merge) # Dataproviders diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 4d71302b3f0..12235e566c4 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -10,6 +10,8 @@ import re import subprocess import tempfile +from six.moves import shlex_quote + from galaxy.datatypes.data import get_file_peek, Text from galaxy.datatypes.metadata import MetadataElement, MetadataParameter from galaxy.datatypes.sniff import iter_headers @@ -148,13 +150,12 @@ class Ipynb(Json): ofilename = ofile_handle.name ofile_handle.close() try: - cmd = 'jupyter nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename) - log.info("Calling command %s" % cmd) - subprocess.call(cmd, shell=True) + cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename] + subprocess.check_call(cmd) ofilename = '%s.html' % ofilename - except: + except subprocess.CalledProcessError: ofilename = dataset.file_name - log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', cmd) + log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd))) return open(ofilename) def set_meta(self, dataset, **kwd): diff --git a/lib/galaxy/external_services/actions.py b/lib/galaxy/external_services/actions.py index 69fd20ca57b..13dc83703d3 100644 --- a/lib/galaxy/external_services/actions.py +++ b/lib/galaxy/external_services/actions.py @@ -1,6 +1,6 @@ # Contains actions that are used in External Services import logging -from urllib import urlopen +import requests from galaxy.web import url_for from galaxy.util.template import fill_template from result_handlers.basic import ExternalServiceActionResultHandler @@ -104,7 +104,7 @@ class ExternalServiceWebAPIActionResult(ExternalServiceResult): @property def content(self): if self._content is None: - self._content = urlopen(self.url).read() + self._content = requests.get(self.url).text return self._content diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index a0e2707c951..b90f9940bcd 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -2,11 +2,10 @@ Module for managing jobs in Pacific Bioscience's SMRT Portal and automatically transferring files produced by SMRT Portal. """ -import json import logging from string import Template -from six.moves.urllib.request import urlopen +import requests from .data_transfer import DataTransfer @@ -88,8 +87,8 @@ class SMRTPortalPlugin(DataTransfer): if self._missing_params(job.params, ['smrt_host', 'smrt_job_id']): return self.job_states.INVALID url = 'http://' + job.params['smrt_host'] + self.api_path + '/Jobs/' + job.params['smrt_job_id'] + '/Status' - r = urlopen(url) - status = json.loads(r.read()) + r = requests.get(url) + status = r.json() # TODO: error handling: unexpected json or bad response, bad url, etc. if status['Code'] == 'Completed': log.debug("SMRT Portal job '%s' is Completed. Initiating transfer." % job.params['smrt_job_id']) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 6dfbc243cd0..bc201e74626 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Need to import pulsar_client absolutel import errno import logging import os +import subprocess from distutils.version import LooseVersion from time import sleep @@ -220,7 +221,7 @@ class PulsarJobRunner(AsynchronousJobRunner): else: log.info("Loading Pulsar app configuration from %s" % pulsar_conf_path) with open(pulsar_conf_path, "r") as f: - conf.update(yaml.load(f) or {}) + conf.update(yaml.safe_load(f) or {}) if "job_metrics_config_file" not in conf: conf["job_metrics"] = self.app.job_metrics if "staging_directory" not in conf: @@ -394,8 +395,7 @@ class PulsarJobRunner(AsynchronousJobRunner): prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None) if prepare_input_files_cmds is not None: for cmd in prepare_input_files_cmds: # run the commands to stage the input files - if 0 != os.system(cmd): - raise Exception('Error running file staging command: %s' % cmd) + subprocess.check_call(cmd, shell=True) job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line def _populate_parameter_defaults(self, job_destination): diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 77af249e5c4..2c717c8f13d 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -119,9 +119,8 @@ def _handle_script_integrity(path, config): sleep_amt = getattr(config, "check_job_script_integrity_sleep", DEFAULT_INTEGRITY_SLEEP) for i in range(count): try: - proc = subprocess.Popen([path], shell=True, env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) - proc.wait() - if proc.returncode == 42: + returncode = subprocess.call([path], env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) + if returncode == 42: script_integrity_verified = True break diff --git a/lib/galaxy/jobs/runners/util/kill.py b/lib/galaxy/jobs/runners/util/kill.py index c0dbd913b13..1a458c63913 100644 --- a/lib/galaxy/jobs/runners/util/kill.py +++ b/lib/galaxy/jobs/runners/util/kill.py @@ -1,6 +1,6 @@ import os +import subprocess from platform import system -from subprocess import Popen from time import sleep try: @@ -41,8 +41,8 @@ def _stock_kill_pid(pid): def __kill_windows(pid): try: - Popen("taskkill /F /T /PID %i" % pid, shell=True) - except Exception: + subprocess.check_call(['taskkill', '/F', '/T', '/PID', pid]) + except subprocess.CalledProcessError: pass diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 01acd6aae8f..8e06024832c 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -9,6 +9,8 @@ import socket import subprocess import threading +from six.moves import shlex_quote + from galaxy.util import listify, sleeper from galaxy.util.json import jsonrpc_request, validate_jsonrpc_response @@ -23,7 +25,7 @@ class TransferManager(object): def __init__(self, app): self.app = app self.sa_session = app.model.context.current - self.command = 'python %s' % os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py')) + self.command = ['python', os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py'))] if app.config.get_bool('enable_job_recovery', True): # Only one Galaxy server process should be able to recover jobs! (otherwise you'll have nasty race conditions) self.running = True @@ -68,9 +70,9 @@ class TransferManager(object): # The transfer script should daemonize fairly quickly - if this is # not the case, this process will need to be moved to a # non-blocking method. - cmd = '%s %s' % (self.command, tj.id) - log.debug('Transfer command is: %s' % cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + cmd = self.command + [tj.id] + log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd))) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.wait() output = p.stdout.read(32768) if p.returncode != 0: diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 7b88b2a3728..5ff70e7c666 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,6 +1,6 @@ import functools import os -import urllib2 +import requests from beaker.cache import CacheManager from beaker.util import parse_cache_config_options @@ -47,10 +47,8 @@ class DoiCache(object): def _raw_get_bibtex(self, doi): dx_url = "http://dx.doi.org/" + doi headers = {'Accept': 'text/bibliography; style=bibtex, application/x-bibtex'} - req = urllib2.Request(dx_url, data="", headers=headers) - response = urllib2.urlopen(req) - bibtex = response.read() - return bibtex + req = requests.get(dx_url, headers=headers) + return req.text def get_bibtex(self, doi): createfunc = functools.partial(self._raw_get_bibtex, doi) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index f832688741e..34a2fc39d74 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -18,6 +18,7 @@ from galaxy.util import ( safe_relpath, string_as_bool, umask_fix_perms, + which, ) from galaxy.util.sleeper import Sleeper @@ -67,10 +68,9 @@ class S3ObjectStore(ObjectStore): self.cache_monitor_thread.start() log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache - try: - subprocess.call('axel') + if which('axel'): self.use_axel = True - except OSError: + else: self.use_axel = False def _configure_connection(self): @@ -341,7 +341,7 @@ class S3ObjectStore(ObjectStore): log.debug("Parallel pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) ncores = multiprocessing.cpu_count() url = key.generate_url(7200) - ret_code = subprocess.call("axel -a -n %s '%s'" % (ncores, url)) + ret_code = subprocess.call(['axel', '-a', '-n', ncores, url]) if ret_code == 0: return True else: diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 92aa6c995ed..d3ec66072b7 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -16,7 +16,7 @@ import time from glob import glob from tempfile import NamedTemporaryFile -from six.moves.urllib.request import urlopen +import requests from galaxy import util from galaxy.util.dictifiable import Dictifiable @@ -340,7 +340,7 @@ class TabularToolDataTable(ToolDataTable, Dictifiable): if filename: tmp_file = NamedTemporaryFile(prefix='TTDT_URL_%s-' % self.name) try: - tmp_file.write(urlopen(filename, timeout=url_timeout).read()) + tmp_file.write(requests.get(filename, timeout=url_timeout).text) except Exception as e: log.error('Error loading Data Table URL "%s": %s', filename, e) continue diff --git a/lib/galaxy/tools/deps/conda_compat.py b/lib/galaxy/tools/deps/conda_compat.py index c45573ef49d..10527bf890d 100644 --- a/lib/galaxy/tools/deps/conda_compat.py +++ b/lib/galaxy/tools/deps/conda_compat.py @@ -61,7 +61,7 @@ def _render_jinja2(recipe_dir): @_Memoized def yamlize(data): - res = yaml.load(data) + res = yaml.safe_load(data) # ensure the result is a dict if res is None: res = {} diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index fc551cb883a..cf215a8f742 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -3,6 +3,7 @@ import collections import logging import os +import subprocess import six @@ -11,7 +12,6 @@ from ..container_resolvers import ( ) from ..docker_util import build_docker_images_command from ..mulled.mulled_build import ( - check_output, DEFAULT_CHANNELS, ensure_installed, InvolucroContext, @@ -54,8 +54,8 @@ CachedV2MulledImageMultiTarget.package_hash = _package_hash def list_docker_cached_mulled_images(namespace=None, hash_func="v2"): command = build_docker_images_command(truncate=True, sudo=False) - command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command - images_and_versions = check_output(command) + images_and_versions = subprocess.check_output(command).strip().split('\n') + images_and_versions = [line.split()[0:2] for line in images_and_versions[1:]] name_filter = get_filter(namespace) def output_line_to_image(line): diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index 5d362f12cd3..381afbe2718 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -116,17 +116,12 @@ def get_affected_packages(args): """ recipes_dir = args.recipes_dir hours = args.diff_hours - cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours) - pkg_list = check_output(cmd, shell=True) - ret = list() - for pkg in pkg_list.strip().split('\n'): + cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours] + changed_files = subprocess.check_output(cmd, cwd=recipes_dir).strip().split('\n') + pkg_list = set([x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')]) + for pkg in pkg_list: if pkg and os.path.exists(os.path.join(recipes_dir, pkg)): - ret.append((get_pkg_name(args, pkg), get_tests(args, pkg))) - return ret - - -def check_output(cmd, shell=True): - return subprocess.check_output(cmd, shell=shell) + yield (get_pkg_name(args, pkg), get_tests(args, pkg)) def conda_versions(pkg_name, file_name): diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py index 23e3021187b..06e0c7afe6f 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py @@ -19,6 +19,7 @@ See recent changes that would be built with: from __future__ import print_function import os +import subprocess import sys import time @@ -27,7 +28,6 @@ from .mulled_build import ( add_build_arguments, args_to_mull_targets_kwds, build_target, - check_output, conda_versions, get_affected_packages, mull_targets, @@ -42,7 +42,13 @@ def _fetch_repo_data(args): repo_data = "%s-repodata.json" % channel if not os.path.exists(repo_data): platform_tag = 'osx-64' if sys.platform == 'darwin' else 'linux-64' - check_output("wget --quiet https://conda.anaconda.org/%s/%s/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, platform_tag, repo_data, repo_data)) + subprocess.check_call([ + 'wget', '--quiet', 'https://conda.anaconda.org/%s/%s/repodata.json.bz2' % (channel, platform_tag), + '-O', '%s.bz2' % repo_data + ]) + subprocess.check_call([ + 'bzip2', '-d', '%s.bz2' % repo_data + ]) return repo_data @@ -55,8 +61,7 @@ def _new_versions(quay, conda): def run_channel(args, build_last_n_versions=1): """Build list of involucro commands (as shell snippet) to run.""" - pkgs = get_affected_packages(args) - for pkg_name, pkg_tests in pkgs: + for pkg_name, pkg_tests in get_affected_packages(args): repo_data = _fetch_repo_data(args) c = conda_versions(pkg_name, repo_data) # only package the most recent N versions diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index e8188e0964a..5c3811024e0 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -97,7 +97,7 @@ class MappableDependencyResolver: @staticmethod def _mapping_file_to_list(mapping_file): with open(mapping_file, "r") as f: - raw_mapping = yaml.load(f) or [] + raw_mapping = yaml.safe_load(f) or [] return map(RequirementMapping.from_dict, raw_mapping) def _expand_mappings(self, requirement): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index c00a636d5a7..c8562f959d4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -15,7 +15,7 @@ import tarfile import tempfile from base64 import b64decode -from six.moves.urllib.request import urlopen +import requests # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. @@ -27,18 +27,16 @@ def url_to_file(url, dest_file): Transfer a file from a remote URL to a temporary file. """ try: - url_reader = urlopen(url) + url_reader = requests.get(url, stream=True) CHUNK = 10 * 1024 # 10k total = 0 fp = open(dest_file, 'wb') - while True: - chunk = url_reader.read(CHUNK) - if not chunk: - break - fp.write(chunk) - total += CHUNK - if total > MAX_SIZE: - break + for chunk in url_reader.iter_content(chunk_size=CHUNK): + if chunk: + fp.write(chunk) + total += CHUNK + if total > MAX_SIZE: + break fp.close() return dest_file except Exception as e: diff --git a/lib/galaxy/tools/locations/dockstore.py b/lib/galaxy/tools/locations/dockstore.py index b379c23c38f..3eb96e55e24 100644 --- a/lib/galaxy/tools/locations/dockstore.py +++ b/lib/galaxy/tools/locations/dockstore.py @@ -57,7 +57,7 @@ class _Ga4ghToolClient(object): if as_string: return descriptor_str else: - return yaml.load(descriptor_str) + return yaml.safe_load(descriptor_str) @property def _requests(self): diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py index 3c79a8b4d7b..db1b582404c 100644 --- a/lib/galaxy/tools/toolbox/parser.py +++ b/lib/galaxy/tools/toolbox/parser.py @@ -59,7 +59,7 @@ class YamlToolConfSource(ToolConfSource): def __init__(self, config_filename): with open(config_filename, "r") as f: - as_dict = yaml.load(f) + as_dict = yaml.safe_load(f) self.as_dict = as_dict def parse_tool_path(self): diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 0511aa29e83..c35d147759c 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -68,7 +68,7 @@ class ToursRegistry(object): tour_id = os.path.splitext(filename)[0] try: with open(tour_path) as handle: - conf = yaml.load(handle) + conf = yaml.safe_load(handle) tour = tour_loader(conf) self.tours[tour_id] = tour_loader(conf) log.info("Loaded tour '%s'" % tour_id) diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 133998f210f..7f6866d6f2d 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -83,4 +83,4 @@ def __read_yaml(path): raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") with open(path, "rb") as f: - return yaml.load(f) + return yaml.safe_load(f) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 2ca160c27f7..8c22e0aaf59 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -67,7 +67,7 @@ def load_app_properties( config_section = "galaxy" with open(config_file, "r") as f: - raw_properties = yaml.load(f) + raw_properties = yaml.safe_load(f) properties = raw_properties[config_section] or {} override_prefix = "%sOVERRIDE_" % config_prefix diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 6410b03ec08..e56449320db 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -111,7 +111,7 @@ class InteractiveEnvironmentRequest(object): raise Exception("[{0}] Could not find allowed_images.yml, or image tag in {0}.ini file for ".format(self.attr.viz_id)) with open(fn, 'r') as handle: - self.allowed_images = [x['image'] for x in yaml.load(handle)] + self.allowed_images = [x['image'] for x in yaml.safe_load(handle)] if len(self.allowed_images) == 0: raise Exception("No allowed images specified for " + self.attr.viz_id) diff --git a/lib/galaxy/web/proxy/__init__.py b/lib/galaxy/web/proxy/__init__.py index 6ff3ee3399f..581050a3d5e 100644 --- a/lib/galaxy/web/proxy/__init__.py +++ b/lib/galaxy/web/proxy/__init__.py @@ -8,9 +8,10 @@ from galaxy.util import sockets from galaxy.util.lazy_process import LazyProcess, NoOpLazyProcess from galaxy.util import sqlite from galaxy.util import unique_id -import urllib2 import time +import requests + log = logging.getLogger(__name__) @@ -301,21 +302,17 @@ class RestGolangProxyIpc(object): 'ContainerIds': container_ids, } - req = urllib2.Request(self.api_url) - req.add_header('Content-Type', 'application/json') - # Sometimes it takes our poor little proxy a second or two to get # going, so if this fails, re-call ourselves with an increased timeout. try: - urllib2.urlopen(req, json.dumps(values)) - except urllib2.URLError as err: - log.debug(err) + requests.get(self.api_url, headers={'Content-Type': 'application/json'}, data=json.dumps(values)) + except requests.exceptions.ConnectionError as err: + log.exception(err) if sleep > 5: excp = "Could not contact proxy after %s seconds" % sum(range(sleep + 1)) raise Exception(excp) time.sleep(sleep) - self.handle_requests(authentication, proxy_requests, route_name, container_ids, sleep=sleep + 1) - pass + self.handle_requests(authentication, proxy_requests, route_name, container_ids, container_interface, sleep=sleep + 1) ProxyMapping = namedtuple('ProxyMapping', ['host', 'port', 'container_ids', 'container_interface']) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 07d0bc03658..685b5abcf56 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -11,8 +11,8 @@ import string import sys import tempfile from textwrap import TextWrapper -import urllib2 +import requests import six from six import StringIO @@ -307,9 +307,8 @@ def _write_option_rst(args, rst, key, heading_level, option_value): def _build_uwsgi_schema(args, app_desc): - req = urllib2.Request('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') - response = urllib2.urlopen(req) - rst_options = response.read() + req = requests.get('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') + rst_options = req.text last_line = None current_opt = None diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 67ec64351f3..63c9e3a78a0 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -282,7 +282,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cre path = trans.app.config.user_preferences_extra_config_file try: with open(path, 'r') as stream: - config = yaml.load(stream) + config = yaml.safe_load(stream) except: log.warning('Config file (%s) could not be found or is malformed.' % path) return {} diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 7db4af7667f..aabb551bdfc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -5,6 +5,8 @@ Upload class import logging import urllib +import requests + from galaxy import jobs, web from galaxy.util import Params from galaxy.util.hash_util import hmac_new @@ -162,8 +164,7 @@ class ASync(BaseUIController): url = "%s%s%s" % (url, url_join_char, urllib.urlencode(params.flatten())) log.debug("connecting to -> %s" % url) trans.log_event("Async connecting to -> %s" % url) - text = urllib.urlopen(url).read(-1) - text = text.strip() + text = requests.get(url).text.strip() if not text.endswith('OK'): raise Exception(text) data.state = data.blurb = data.states.RUNNING diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 365dd48c6b9..2a1ac0a09dc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1,4 +1,5 @@ import glob +import json import logging import operator import os @@ -8,10 +9,9 @@ import sys import tarfile import tempfile import urllib -import urllib2 import zipfile -from json import dumps, loads +import requests from markupsafe import escape from sqlalchemy import and_, false from sqlalchemy.orm import eagerload_all @@ -555,7 +555,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta if len(em_string): payload = None try: - payload = loads(em_string) + payload = json.loads(em_string) except Exception: message = 'Invalid JSON input' status = 'error' @@ -1122,8 +1122,8 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta json_file_path = upload_common.create_paramfile(trans, uploaded_datasets) data_list = [ud.data for ud in uploaded_datasets] job_params = {} - job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files')) - job_params['uuid'] = dumps(kwd.get('uuid', None)) + job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files')) + job_params['uuid'] = json.dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params) trans.sa_session.add(job) trans.sa_session.flush() @@ -2762,9 +2762,7 @@ def lucene_search(trans, cntrller, search_term, search_url, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') full_url = "%s/find?%s" % (search_url, urllib.urlencode({"kwd" : search_term})) - response = urllib2.urlopen(full_url) - ldda_ids = loads(response.read())["ids"] - response.close() + ldda_ids = requests.get(full_url).json()['ids'] lddas = [trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(ldda_id) for ldda_id in ldda_ids] return status, message, get_sorted_accessible_library_items(trans, cntrller, lddas, 'name') diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 4817aee5b84..a8d2df5f953 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -3,8 +3,8 @@ Contains the main interface in the Universe class """ import cgi import os -import urllib +import requests from paste.httpexceptions import HTTPNotFound, HTTPBadGateway from galaxy import web @@ -470,8 +470,8 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): def bucket_proxy(self, trans, bucket=None, **kwd): if bucket: trans.response.set_content_type('text/xml') - b_list_xml = urllib.urlopen('http://s3.amazonaws.com/%s/' % bucket) - return b_list_xml.read() + b_list_xml = requests.get('http://s3.amazonaws.com/%s/' % bucket) + return b_list_xml.text raise Exception("You must specify a bucket") # ---- Debug methods ---------------------------------------------------- diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index f134ebe48b4..9c7c1e5424e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -961,7 +961,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization continue with open(image_file, 'r') as handle: - self.gie_image_map[gie] = yaml.load(handle) + self.gie_image_map[gie] = yaml.safe_load(handle) return trans.fill_template_mako( "visualization/gie.mako", diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index d5216798bed..55706557e22 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -4,7 +4,7 @@ import json import logging import os import sgmllib -import urllib2 +import requests from sqlalchemy import and_ from sqlalchemy.orm import joinedload @@ -763,7 +763,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi # Load workflow from external URL # NOTE: blocks the web thread. try: - workflow_data = urllib2.urlopen(url).read() + workflow_data = requests.get(url).text except Exception as e: message = "Failed to open URL: %s. Exception: %s" % (escape(url), escape(str(e))) status = 'error' diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index 6359c1defd8..5c325c2239e 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -1,5 +1,6 @@ import logging import os +import subprocess from datetime import datetime, timedelta from decimal import Decimal @@ -148,12 +149,11 @@ class System(BaseUIController): message=message) def get_disk_usage(self, file_path): - df_cmd = 'df -h ' + file_path is_sym_link = os.path.islink(file_path) file_system = disk_size = disk_used = disk_avail = disk_cap_pct = mount = None - df_file = os.popen(df_cmd) - while True: - df_line = df_file.readline() + df_output = subprocess.check_output(['df', '-h', file_path]) + + for df_line in df_output: df_line = df_line.strip() if df_line: df_line = df_line.lower() @@ -176,7 +176,6 @@ class System(BaseUIController): pass else: break # EOF - df_file.close() return (file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount) @web.expose diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index 1695cf69507..2fa3d3be406 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -3,7 +3,8 @@ import os import shutil import tarfile import tempfile -import urllib + +import requests from galaxy import util from galaxy import web @@ -74,7 +75,7 @@ class UploadController(BaseUIController): elif url: valid_url = True try: - stream = urllib.urlopen(url) + stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % str(e) @@ -83,11 +84,9 @@ class UploadController(BaseUIController): if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') - while 1: - chunk = stream.read(util.CHUNK_SIZE) - if not chunk: - break - uploaded_file.write(chunk) + for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): + if chunk: + uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 diff --git a/lib/galaxy/webhooks/__init__.py b/lib/galaxy/webhooks/__init__.py index 7e9c9482433..6cc2c34d9d9 100644 --- a/lib/galaxy/webhooks/__init__.py +++ b/lib/galaxy/webhooks/__init__.py @@ -64,7 +64,7 @@ class WebhooksRegistry(object): def load_webhook_from_config(self, config_dir, config_file): try: with open(os.path.join(config_dir, config_file)) as file: - config = yaml.load(file) + config = yaml.safe_load(file) path = os.path.normpath(os.path.join(config_dir, '..')) webhook = Webhook( config['name'], diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 422d663c4a9..5daa0b21c04 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -7,7 +7,7 @@ import tempfile import threading from time import gmtime, strftime -from six.moves.urllib.request import urlopen +import requests from sqlalchemy import and_, false import tool_shed.repository_types.util as rt_util @@ -809,24 +809,20 @@ class ImportRepositoryManager(object): uploaded_file=None, capsule_file_name=None) if url: - valid_url = True try: - stream = urlopen(url) + stream = requests.get(url, stream=True) except Exception as e: - valid_url = False return_dict['error_message'] = 'Error importing file via http: %s' % str(e) return_dict['status'] = 'error' return return_dict - if valid_url: - fd, uploaded_file_name = tempfile.mkstemp() - uploaded_file = open(uploaded_file_name, 'wb') - while 1: - chunk = stream.read(CHUNK_SIZE) - if not chunk: - break + + fd, uploaded_file_name = tempfile.mkstemp() + uploaded_file = open(uploaded_file_name, 'wb') + for chunk in stream.iter_content(chunk_size=CHUNK_SIZE): + if chunk: uploaded_file.write(chunk) - uploaded_file.flush() - uploaded_file_filename = url.split('/')[-1] + uploaded_file.flush() + uploaded_file_filename = url.split('/')[-1] elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name diff --git a/scripts/data_libraries/build_lucene_index.py b/scripts/data_libraries/build_lucene_index.py index a53ec522163..b3f787e8b02 100644 --- a/scripts/data_libraries/build_lucene_index.py +++ b/scripts/data_libraries/build_lucene_index.py @@ -14,7 +14,8 @@ import csv import os import sys import urllib -import urllib2 + +import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) @@ -39,9 +40,7 @@ def main(ini_file): def build_index(search_url, dataset_file): url = "%s/index?%s" % (search_url, urllib.urlencode({"docfile": dataset_file})) - request = urllib2.Request(url) - request.get_method = lambda: "PUT" - urllib2.urlopen(request) + requests.put(url) def create_dataset_file(dataset_iter): diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index 7858d9f5abb..f483db8c1f5 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -16,9 +16,10 @@ from __future__ import print_function import os import sys -import urllib2 from xml import etree +import requests + sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) import galaxy.model @@ -35,7 +36,7 @@ EDAM_OWL_URL = "http://data.bioontology.org/ontologies/EDAM/submissions/25/downl if not os.path.exists("/tmp/edam.owl"): - open("/tmp/edam.owl", "w").write(urllib2.urlopen(EDAM_OWL_URL).read()) + open("/tmp/edam.owl", "w").write(requests.get(EDAM_OWL_URL).text) owl_xml_tree = etree.ElementTree.parse("/tmp/edam.owl") diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 4619c6929c6..da8048522f2 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -191,11 +191,11 @@ def main(argv): annotate('init_start', 'Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) annotate('init_end') REPORT_DIR = args.report_directory diff --git a/scripts/grt/upload.py b/scripts/grt/upload.py index 332cbe89e6d..a93efb86a62 100644 --- a/scripts/grt/upload.py +++ b/scripts/grt/upload.py @@ -28,11 +28,11 @@ def main(argv): logging.info('Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) REPORT_DIR = args.report_directory GRT_URL = config['grt']['url'].rstrip('/') + '/' diff --git a/scripts/microbes/harvest_bacteria.py b/scripts/microbes/harvest_bacteria.py index 7161669b5c9..3226dec78ce 100644 --- a/scripts/microbes/harvest_bacteria.py +++ b/scripts/microbes/harvest_bacteria.py @@ -8,9 +8,9 @@ import os import sys import time from ftplib import FTP -from urllib2 import urlopen from urllib import urlretrieve +import requests from BeautifulSoup import BeautifulSoup from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark @@ -26,7 +26,7 @@ desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_Gen # number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="): - for row in BeautifulSoup(urlopen(url)).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]): + for row in BeautifulSoup(requests.get(url).text).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]): row = str(row).replace("\n", "").replace("\r", "") fields = row.split("") @@ -65,7 +65,7 @@ def get_chroms_by_project_id(org_num, base_url="http://www.ncbi.nlm.nih.gov/entr html_count += 1 url = "%s%s" % (base_url, org_num) try: - html = urlopen(url) + html = requests.get(url).text except: print "GENOME PROJECT FAILED:", html_count, "org:", org_num, url html = None diff --git a/scripts/tool_shed/api/export.py b/scripts/tool_shed/api/export.py index a198fffd966..6a5d030c6a0 100644 --- a/scripts/tool_shed/api/export.py +++ b/scripts/tool_shed/api/export.py @@ -11,7 +11,8 @@ import argparse import os import sys import tempfile -import urllib2 + +import requests sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) from tool_shed.util import basic_util @@ -98,24 +99,11 @@ def main(options): download_url = export_dict['download_url'] download_dir = os.path.abspath(options.download_dir) file_path = os.path.join(download_dir, repositories_archive_filename) - src = None - dst = None - try: - src = urllib2.urlopen(download_url) - dst = open(file_path, 'wb') - while True: - chunk = src.read(CHUNK_SIZE) + src = requests.get(download_url, stream=True) + with open(file_path, 'wb') as dst: + for chunk in src.iter_content(chunk_size=CHUNK_SIZE): if chunk: dst.write(chunk) - else: - break - except: - raise - finally: - if src: - src.close() - if dst: - dst.close() print "Successfully exported revision ", options.changeset_revision, " of repository ", options.name, " owned by ", options.owner print "to location ", file_path else: diff --git a/scripts/transfer.py b/scripts/transfer.py index 8bcb23f432e..3938ca48846 100644 --- a/scripts/transfer.py +++ b/scripts/transfer.py @@ -215,6 +215,7 @@ def transfer(app, transfer_job_id): def http_transfer(transfer_job): """Plugin" for handling http(s) transfers.""" url = transfer_job.params['url'] + assert url.startswith('http://') or url.startswith('https://') try: f = urllib2.urlopen(url) except urllib2.URLError as e: diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 79a0c1c2218..f3c45c03368 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -172,7 +172,7 @@ class BaseWorkflowsApiTestCase(api.ApiTestCase): ) if jobs_descriptions is None: assert source_type != "path" - jobs_descriptions = yaml.load(has_workflow) + jobs_descriptions = yaml.safe_load(has_workflow) test_data = jobs_descriptions.get("test_data", {}) diff --git a/test/api/test_workflows_from_yaml.py b/test/api/test_workflows_from_yaml.py index 9beddf7aa2e..2b39628c27d 100644 --- a/test/api/test_workflows_from_yaml.py +++ b/test/api/test_workflows_from_yaml.py @@ -55,11 +55,11 @@ steps: assert tool_count['random_lines1'] == 1 assert tool_count['cat1'] == 2 -# FIXME: This test fails on some machines due to (we're guessing) yaml loading +# FIXME: This test fails on some machines due to (we're guessing) yaml.safe_loading # order being not guaranteed and inconsistent across platforms. The workflow -# yaml loader probably needs to enforce order using something like the +# yaml.safe_loader probably needs to enforce order using something like the # approach described here: -# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml-load +# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml.safe_load # def test_multiple_input( self ): # history_id = self.dataset_populator.new_history() # self._run_jobs(""" diff --git a/test/base/workflows_format_2/converter.py b/test/base/workflows_format_2/converter.py index 05a9a319c49..c35ec2797c0 100644 --- a/test/base/workflows_format_2/converter.py +++ b/test/base/workflows_format_2/converter.py @@ -32,7 +32,7 @@ RUN_ACTIONS_TO_STEPS = { def yaml_to_workflow(has_yaml, galaxy_interface, workflow_directory): """Convert a Format 2 workflow into standard Galaxy format from supplied stream.""" - as_python = yaml.load(has_yaml) + as_python = yaml.safe_load(has_yaml) return python_to_workflow(as_python, galaxy_interface, workflow_directory) @@ -109,7 +109,7 @@ def _python_to_workflow(as_python, conversion_context): run_action_path = run_action["@import"] runnable_path = os.path.join(conversion_context.workflow_directory, run_action_path) with open(runnable_path, "r") as f: - runnable_description = yaml.load(f) + runnable_description = yaml.safe_load(f) run_action = runnable_description run_class = run_action["class"] diff --git a/test/base/workflows_format_2/main.py b/test/base/workflows_format_2/main.py index 70d59a8db8d..a4af30e8da6 100644 --- a/test/base/workflows_format_2/main.py +++ b/test/base/workflows_format_2/main.py @@ -20,7 +20,7 @@ def convert_and_import_workflow(has_workflow, **kwds): if workflow_directory is None: workflow_directory = os.path.dirname(has_workflow) with open(workflow_path, "r") as f: - has_workflow = yaml.load(f) + has_workflow = yaml.safe_load(f) if workflow_directory is not None: workflow_directory = os.path.abspath(workflow_directory) diff --git a/test/functional/webhooks/phdcomics/helper/__init__.py b/test/functional/webhooks/phdcomics/helper/__init__.py index b6828d2c21a..d3ec4d7fbe5 100644 --- a/test/functional/webhooks/phdcomics/helper/__init__.py +++ b/test/functional/webhooks/phdcomics/helper/__init__.py @@ -1,7 +1,7 @@ -import urllib -import re -import random import logging +import random +import re +import urllib log = logging.getLogger(__name__) diff --git a/test/galaxy_selenium/data.py b/test/galaxy_selenium/data.py index a3d969f8a4f..ef15e27197b 100644 --- a/test/galaxy_selenium/data.py +++ b/test/galaxy_selenium/data.py @@ -3,4 +3,4 @@ from pkg_resources import resource_string import yaml data_yaml = resource_string(__name__, 'navigation-data.yml').decode("UTF-8") -NAVIGATION_DATA = yaml.load(data_yaml) +NAVIGATION_DATA = yaml.safe_load(data_yaml) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index bb5f93d8274..8981b3779da 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -819,7 +819,7 @@ class NavigatesGalaxy(HasDriver): self.home() with open(path, "r") as f: - tour_dict = yaml.load(f) + tour_dict = yaml.safe_load(f) steps = tour_dict["steps"] for i, step in enumerate(steps): title = step.get("title", None) diff --git a/test/unit/workflows/workflow_support.py b/test/unit/workflows/workflow_support.py index ee04ded4fc9..6f4f567cb66 100644 --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -75,7 +75,7 @@ class TestToolbox(object): def yaml_to_model(has_dict, id_offset=100): if isinstance(has_dict, str): - has_dict = yaml.load(has_dict) + has_dict = yaml.safe_load(has_dict) workflow = model.Workflow() workflow.steps = []