From cc336f61c09eac6c3dd9dba706b08481dc5bd709 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 3 Jun 2016 18:12:17 +0100 Subject: [PATCH] Make some files compatible with Python3 Also enlarge the set of files tested with flake8 under Python3 --- .ci/py3_sources.txt | 52 ++++++- contrib/galaxy_config_merger.py | 11 +- contrib/nagios/check_galaxy.py | 17 +-- cron/build_chrom_db.py | 16 ++- cron/parse_builds.py | 21 +-- cron/parse_builds_3_sites.py | 16 ++- lib/galaxy/config.py | 15 ++- lib/galaxy_utils/sequence/fastq.py | 54 ++++---- lib/tool_shed/util/container_util.py | 6 +- .../admin_cleanup_datasets.py | 50 +++---- scripts/cleanup_datasets/cleanup_datasets.py | 127 +++++++++--------- test/api/test_workflows_from_yaml.py | 6 +- test/base/interactor.py | 46 ++++--- test/base/twilltestcase.py | 25 ++-- test/unit/tools/test_actions.py | 1 - test/unit/workflows/test_run_parameters.py | 3 +- tool_list.py | 12 +- tools/data_source/data_source.py | 10 +- tools/data_source/fetch.py | 8 +- tools/data_source/genbank.py | 8 +- tools/data_source/hbvar_filter.py | 9 +- tools/data_source/import.py | 9 +- tools/data_source/microbial_import.py | 13 +- tools/data_source/microbial_import_code.py | 10 +- tools/data_source/upload.py | 22 +-- 25 files changed, 325 insertions(+), 242 deletions(-) diff --git a/.ci/py3_sources.txt b/.ci/py3_sources.txt index 69f7f69dc08..325fa99e160 100644 --- a/.ci/py3_sources.txt +++ b/.ci/py3_sources.txt @@ -1,5 +1,19 @@ -lib/galaxy/util/ -lib/galaxy/jobs/runners/util/ +contrib/ +cron/ +lib/galaxy/actions/ +lib/galaxy/auth/ +lib/galaxy/config.py +lib/galaxy/dependencies/ +lib/galaxy/eggs/ +lib/galaxy/exceptions/ +lib/galaxy/external_services/ +lib/galaxy/forms/ +lib/galaxy/jobs/ +lib/galaxy/objectstore/ +lib/galaxy/openid/ +lib/galaxy/quota/ +lib/galaxy/sample_tracking/ +lib/galaxy/tags/ lib/galaxy/tools/cwl/ lib/galaxy/tools/parser/ lib/galaxy/tools/lint.py @@ -10,8 +24,38 @@ lib/galaxy/tools/linters/ lib/galaxy/tools/deps/ lib/galaxy/tools/toolbox/ lib/galaxy/tools/parser/ -lib/galaxy/jobs/metrics/ -lib/galaxy/objectstore/ +lib/galaxy/tours/ +lib/galaxy/util/ +lib/galaxy/work/ +lib/galaxy_ext/ +lib/galaxy_utils/ +lib/log_tempfile.py +lib/psyco_full.py +lib/tool_shed/capsule/ +lib/tool_shed/dependencies/ +lib/tool_shed/grids/ +lib/tool_shed/managers/ +lib/tool_shed/metadata/ +lib/tool_shed/repository_types/ +lib/tool_shed/tools/ +lib/tool_shed/util/ +lib/tool_shed/utility_containers/ scripts/api/common.py scripts/api/display.py scripts/api/workflow_execute_parameters.py +scripts/auth/ +scripts/cleanup_datasets/admin_cleanup_datasets.py +scripts/cleanup_datasets/cleanup_datasets.py +test/api/test_workflows_from_yaml.py +test/base/ +test/casperjs/ +test/functional/ +test/integration/ +test/manual/ +test/unit/tools/test_actions.py +test/unit/workflows/test_run_parameters.py +tool_list.py +tools/data_source/ +tools/evolution/ +tools/sr_mapping/ +tools/visualization/ diff --git a/contrib/galaxy_config_merger.py b/contrib/galaxy_config_merger.py index 7176e3e080d..5e2c14938ee 100644 --- a/contrib/galaxy_config_merger.py +++ b/contrib/galaxy_config_merger.py @@ -23,11 +23,14 @@ THE ORIGINAL WORK IS WITH YOU. Script for merging specific local Galaxy config galaxy.ini.cri with default Galaxy galaxy.ini.sample ''' -import ConfigParser +from __future__ import print_function + import logging import optparse import sys +from six.moves import configparser + def main(): # logging configuration @@ -42,15 +45,15 @@ def main(): for option in ['sample', 'config']: if getattr(options, option) is None: - print "Please supply a --%s parameter.\n" % (option) + print("Please supply a --%s parameter.\n" % (option)) parser.print_help() sys.exit() - config_sample = ConfigParser.RawConfigParser() + config_sample = configparser.RawConfigParser() config_sample.read(options.sample) config_sample_content = open(options.sample, 'r').read() - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() config.read(options.config) logging.info("Merging your own config file %s into the sample one %s." % (options.config, options.sample)) diff --git a/contrib/nagios/check_galaxy.py b/contrib/nagios/check_galaxy.py index 200f3e1a8e9..75abb60fcd0 100755 --- a/contrib/nagios/check_galaxy.py +++ b/contrib/nagios/check_galaxy.py @@ -3,6 +3,7 @@ check_galaxy can be run by hand, although it is meant to run from cron via the check_galaxy.sh script in Galaxy's cron/ directory. """ +from __future__ import print_function import filecmp import formatter @@ -63,7 +64,7 @@ def usage(): try: opts, args = getopt.getopt( sys.argv[1:], 'n' ) except getopt.GetoptError as e: - print str(e) + print(str(e)) usage() if len( args ) < 1: usage() @@ -75,7 +76,7 @@ new_history = False for o, a in opts: if o == "-n": if debug: - print "Specified -n, will create a new history" + print("Specified -n, will create a new history") new_history = True else: usage() @@ -83,7 +84,7 @@ for o, a in opts: # state information var_dir = os.path.join( os.path.expanduser('~'), ".check_galaxy", server ) if not os.access( var_dir, os.F_OK ): - os.makedirs( var_dir, 0700 ) + os.makedirs( var_dir, 0o700 ) # default timeout for twill browser is never socket.setdefaulttimeout(300) @@ -256,7 +257,7 @@ class Browser: def check_state(self): if self.hda_state != "ok": self.get("/datasets/%s/stderr" % self.hda_id) - print tc.browser.get_html() + print(tc.browser.get_html()) raise Exception("HDA %s NOT OK: %s" % (self.hda_id, self.hda_state)) def diff(self): @@ -281,7 +282,7 @@ class Browser: self.get('/datasets/%s/delete' % hda['id']) hdas = [hda['id'] for hda in self.undeleted_hdas] if hdas: - print "Remaining datasets ids:", " ".join(hdas) + print("Remaining datasets ids:", " ".join(hdas)) raise Exception("History still contains datasets after attempting to delete them") def check_if_logged_in(self): @@ -344,7 +345,7 @@ class loggedinParser(htmllib.HTMLParser): def dprint(str): if debug: - print str + print(str) # do stuff here if __name__ == "__main__": @@ -360,7 +361,7 @@ if __name__ == "__main__": dprint("not logged in... logging in") b.login(username, password) - for tool, params in tools.iteritems(): + for tool, params in tools.items(): check_file = "" @@ -388,5 +389,5 @@ if __name__ == "__main__": b.diff() b.delete_datasets() - print "OK" + print("OK") sys.exit(0) diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index d148e05d0a3..4cb7f34c895 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - """ Connects to a UCSC table browser and scrapes chrominfo for every build specified by an input file (such as one output by parse_builds.py). @@ -12,18 +11,21 @@ All chromInfo is placed in a path with the convention Usage: python build_chrom_db.py dbpath/ [builds_file] """ +from __future__ import print_function import fileinput import os import sys -import urllib + +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import urlopen import parse_builds def getchrominfo(url, db): tableURL = "http://genome-test.cse.ucsc.edu/cgi-bin/hgTables?" - URL = tableURL + urllib.urlencode({ + URL = tableURL + urlencode({ "clade": "", "org": "", "db": db, @@ -34,7 +36,7 @@ def getchrominfo(url, db): "hgta_regionType": "", "position": "", "hgta_doTopSubmit": "get info"}) - page = urllib.urlopen(URL) + page = urlopen(URL) for line in page: line = line.rstrip( "\r\n" ) if line.startswith("#"): @@ -68,12 +70,12 @@ if __name__ == "__main__": for build in builds: if build == "?": continue # no lengths for unspecified chrom - print "Retrieving " + build + print("Retrieving " + build) outfile_name = dbpath + build + ".len" try: with open(outfile_name, "w") as outfile: for chrominfo in getchrominfo("http://genome-test.cse.ucsc.edu/cgi-bin/hgTables?", build): - print >> outfile, "\t".join(chrominfo) + print("\t".join(chrominfo), file=outfile) except Exception as e: - print "Failed to retrieve %s: %s" % (build, e) + print("Failed to retrieve %s: %s" % (build, e)) os.remove(outfile_name) diff --git a/cron/parse_builds.py b/cron/parse_builds.py index 504ff4fa832..06566e55cb0 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -1,34 +1,35 @@ #!/usr/bin/env python - """ Connects to the URL specified and outputs builds available at that DSN in tabular format. UCSC Main gateway is used as default. build description """ +from __future__ import print_function import sys -import urllib import xml.etree.ElementTree as ElementTree +from six.moves.urllib.request import urlopen + def getbuilds(url): try: - page = urllib.urlopen(url) + page = urlopen(url) except: - print "#Unable to open " + url - print "?\tunspecified (?)" + print("#Unable to open " + url) + print("?\tunspecified (?)") sys.exit(1) text = page.read() try: tree = ElementTree.fromstring(text) except: - print "#Invalid xml passed back from " + url - print "?\tunspecified (?)" + print("#Invalid xml passed back from " + url) + print("?\tunspecified (?)") sys.exit(1) - print "#Harvested from " + url - print "?\tunspecified (?)" + print("#Harvested from " + url) + print("?\tunspecified (?)") for dsn in tree: build = dsn.find("SOURCE").attrib['id'] description = dsn.find("DESCRIPTION").text.replace(" - Genome at UCSC", "").replace(" Genome at UCSC", "") @@ -49,4 +50,4 @@ if __name__ == "__main__": else: URL = "http://genome.cse.ucsc.edu/cgi-bin/das/dsn" for build in getbuilds(URL): - print build[0] + "\t" + build[1] + " (" + build[0] + ")" + print(build[0] + "\t" + build[1] + " (" + build[0] + ")") diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index 7a5e5d06fca..2a9e41533c3 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -2,10 +2,12 @@ """ Connects to sites and determines which builds are available at each. """ +from __future__ import print_function -import urllib import xml.etree.ElementTree as ElementTree +from six.moves.urllib.request import urlopen + sites = ['http://genome.ucsc.edu/cgi-bin/', 'http://archaea.ucsc.edu/cgi-bin/', 'http://genome-test.cse.ucsc.edu/cgi-bin/'] @@ -18,17 +20,17 @@ def main(): trackurl = sites[i] + "hgTracks?" builds = [] try: - page = urllib.urlopen(site) + page = urlopen(site) except: - print "#Unable to connect to " + site + print("#Unable to connect to " + site) continue text = page.read() try: tree = ElementTree.fromstring(text) except: - print "#Invalid xml passed back from " + site + print("#Invalid xml passed back from " + site) continue - print "#Harvested from", site + print("#Harvested from", site) for dsn in tree: build = dsn.find("SOURCE").attrib['id'] @@ -36,9 +38,9 @@ def main(): build_dict = {} for build in builds: build_dict[build] = 0 - builds = build_dict.keys() + builds = list(build_dict.keys()) yield [names[i], trackurl, builds] if __name__ == "__main__": for site in main(): - print site[0] + "\t" + site[1] + "\t" + ",".join(site[2]) + print(site[0] + "\t" + site[1] + "\t" + ",".join(site[2])) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 31741c73293..50ae0dccff9 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -4,7 +4,6 @@ Universe configuration builder. # absolute_import needed for tool_shed package. from __future__ import absolute_import -import ConfigParser import logging import logging.config import os @@ -16,7 +15,9 @@ import sys import tempfile import threading from datetime import timedelta + from six import string_types +from six.moves import configparser from galaxy.exceptions import ConfigurationError from galaxy.util import listify @@ -57,7 +58,7 @@ class Configuration( object ): self.__parse_config_file_options( kwargs ) # Collect the umask and primary gid from the environment - self.umask = os.umask( 077 ) # get the current umask + self.umask = os.umask( 0o77 ) # get the current umask os.umask( self.umask ) # can't get w/o set, so set it back self.gid = os.getgid() # if running under newgrp(1) we'll need to fix the group of data created on the cluster @@ -356,7 +357,7 @@ class Configuration( object ): self.irods_default_resource = kwargs.get( 'irods_default_resource', None ) # Parse global_conf and save the parser global_conf = kwargs.get( 'global_conf', None ) - global_conf_parser = ConfigParser.ConfigParser() + global_conf_parser = configparser.ConfigParser() self.config_file = None self.global_conf_parser = global_conf_parser if global_conf and "__file__" in global_conf: @@ -415,7 +416,7 @@ class Configuration( object ): self.amqp = {} try: amqp_config = global_conf_parser.items("galaxy_amqp") - except ConfigParser.NoSectionError: + except configparser.NoSectionError: amqp_config = {} for k, v in amqp_config: self.amqp[k] = v @@ -613,7 +614,7 @@ class Configuration( object ): rval[ tool ].append( runner_dict ) return rval - except ConfigParser.NoSectionError: + except configparser.NoSectionError: return {} def get( self, key, default ): @@ -685,7 +686,7 @@ class Configuration( object ): def guess_galaxy_port(self): # Code derived from IPython work ie.mako - config = ConfigParser.SafeConfigParser({'port': '8080'}) + config = configparser.SafeConfigParser({'port': '8080'}) if self.config_file: config.read( self.config_file ) @@ -734,7 +735,7 @@ def get_database_engine_options( kwargs, model_prefix='' ): prefix = "%sdatabase_engine_option_" % model_prefix prefix_len = len( prefix ) rval = {} - for key, value in kwargs.iteritems(): + for key, value in kwargs.items(): if key.startswith( prefix ): key = key[prefix_len:] if key in conversions: diff --git a/lib/galaxy_utils/sequence/fastq.py b/lib/galaxy_utils/sequence/fastq.py index aa3be1187b4..5a7cc3cc340 100644 --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -1,10 +1,14 @@ # Dan Blankenberg +from __future__ import print_function + import math import string -import transform -from six import string_types -from sequence import SequencingRead -from fasta import fastaSequence + +from six import Iterator, string_types + +from . import transform +from .fasta import fastaSequence +from .sequence import SequencingRead class fastqSequencingRead( SequencingRead ): @@ -27,19 +31,19 @@ class fastqSequencingRead( SequencingRead ): if score <= 0: # can't take log10( 1 - 1 ); make <= 0 into -5 return -5 return int( round( 10.0 * math.log10( math.pow( 10.0, ( float( score ) / 10.0 ) ) - 1.0 ) ) ) - return map( phred_to_solexa, decimal_score_list ) + return [phred_to_solexa(_) for _ in decimal_score_list] @classmethod def convert_score_solexa_to_phred( cls, decimal_score_list ): def solexa_to_phred( score ): return int( round( 10.0 * math.log10( math.pow( 10.0, ( float( score ) / 10.0 ) ) + 1.0 ) ) ) - return map( solexa_to_phred, decimal_score_list ) + return [solexa_to_phred(_) for _ in decimal_score_list] @classmethod def restrict_scores_to_valid_range( cls, decimal_score_list ): def restrict_score( score ): return max( min( score, cls.quality_max ), cls.quality_min ) - return map( restrict_score, decimal_score_list ) + return [restrict_score(_) for _ in decimal_score_list] @classmethod def transform_scores_to_valid_range( cls, decimal_score_list): @@ -180,7 +184,7 @@ class fastqSequencingRead( SequencingRead ): if self.is_ascii_encoded(): new_read.quality = self.quality[left_column_offset:right_column_offset] else: - quality = map( str, self.get_decimal_quality_scores()[left_column_offset:right_column_offset] ) + quality = [str(_) for _ in self.get_decimal_quality_scores()[left_column_offset:right_column_offset]] if quality: new_read.quality = "%s " % " ".join( quality ) else: @@ -271,7 +275,7 @@ class fastqCSSangerRead( fastqSequencingRead ): quality_max = 93 score_system = 'phred' sequence_space = 'color' - valid_sequence_list = map( str, range( 7 ) ) + [ '.' ] + valid_sequence_list = [str(_) for _ in range(7)] + [ '.' ] def __len__( self ): if self.has_adapter_base(): # Adapter base is not counted in length of read @@ -366,7 +370,7 @@ for format in [ fastqIlluminaRead, fastqSolexaRead, fastqSangerRead, fastqCSSang class fastqAggregator( object ): - VALID_FORMATS = FASTQ_FORMATS.keys() + VALID_FORMATS = list(FASTQ_FORMATS.keys()) def __init__( self ): self.ascii_values_used = [] # quick lookup of all ascii chars used @@ -444,7 +448,7 @@ class fastqAggregator( object ): return self.nuc_index_base[ column ] def get_score_list_for_column( self, column ): - return self.nuc_index_quality[ column ].keys() + return list(self.nuc_index_quality[ column ].keys()) def get_score_min_for_column( self, column ): return min( self.nuc_index_quality[ column ].keys() ) @@ -453,7 +457,7 @@ class fastqAggregator( object ): return max( self.nuc_index_quality[ column ].keys() ) def get_score_sum_for_column( self, column ): - return sum( score * count for score, count in self.nuc_index_quality[ column ].iteritems() ) + return sum( score * count for score, count in self.nuc_index_quality[ column ].items() ) def get_score_at_position_for_column( self, column, position ): score_value_dict = self.nuc_index_quality[ column ] @@ -529,7 +533,7 @@ class fastqAggregator( object ): return column_stats -class fastqReader( object ): +class fastqReader( Iterator ): def __init__( self, fh, format='sanger', apply_galaxy_conventions=False ): self.file = fh self.format = format @@ -538,7 +542,7 @@ class fastqReader( object ): def close( self ): return self.file.close() - def next(self): + def __next__(self): while True: fastq_header = self.file.readline() if not fastq_header: @@ -572,7 +576,7 @@ class fastqReader( object ): def __iter__( self ): while True: - yield self.next() + yield next(self) class ReadlineCountFile( object ): @@ -595,27 +599,27 @@ class fastqVerboseErrorReader( fastqReader ): super( fastqVerboseErrorReader, self ).__init__( ReadlineCountFile( fh ), **kwds ) self.last_good_identifier = None - def next( self ): + def __next__( self ): last_good_end_offset = self.file.tell() last_readline_count = self.file.readline_count try: - block = super( fastqVerboseErrorReader, self ).next() + block = super( fastqVerboseErrorReader, self ).__next__() self.last_good_identifier = block.identifier return block except StopIteration as e: raise e except Exception as e: - print "There was an error reading your input file. Your input file is likely malformed.\nIt is suggested that you double-check your original input file for errors -- helpful information for this purpose has been provided below.\nHowever, if you think that you have encountered an actual error with this tool, please do tell us by using the bug reporting mechanism.\n\nThe reported error is: '%s'." % e + print("There was an error reading your input file. Your input file is likely malformed.\nIt is suggested that you double-check your original input file for errors -- helpful information for this purpose has been provided below.\nHowever, if you think that you have encountered an actual error with this tool, please do tell us by using the bug reporting mechanism.\n\nThe reported error is: '%s'." % e) if self.last_good_identifier is not None: - print "The last valid FASTQ read had an identifier of '%s'." % self.last_good_identifier + print("The last valid FASTQ read had an identifier of '%s'." % self.last_good_identifier) else: - print "The error occurred at the start of your file and no valid FASTQ reads were found." + print("The error occurred at the start of your file and no valid FASTQ reads were found.") error_offset = self.file.tell() error_byte_count = error_offset - last_good_end_offset print_error_bytes = min( self.MAX_PRINT_ERROR_BYTES, error_byte_count ) - print "The error in your file occurs between lines '%i' and '%i', which corresponds to byte-offsets '%i' and '%i', and contains the text (%i of %i bytes shown):\n" % ( last_readline_count + 1, self.file.readline_count, last_good_end_offset, error_offset, print_error_bytes, error_byte_count ) + print("The error in your file occurs between lines '%i' and '%i', which corresponds to byte-offsets '%i' and '%i', and contains the text (%i of %i bytes shown):\n" % ( last_readline_count + 1, self.file.readline_count, last_good_end_offset, error_offset, print_error_bytes, error_byte_count )) self.file.seek( last_good_end_offset ) - print self.file.read( print_error_bytes ) + print(self.file.read( print_error_bytes )) raise e @@ -646,14 +650,14 @@ class fastqNamedReader( object ): if not self.offset_dict[ sequence_id ]: del self.offset_dict[ sequence_id ] self.file.seek( seq_offset ) - rval = self.reader.next() + rval = next(self.reader) # assert rval.id == sequence_id, 'seq id mismatch' #should be able to remove this self.file.seek( initial_offset ) else: while True: offset = self.file.tell() try: - fastq_read = self.reader.next() + fastq_read = next(self.reader) except StopIteration: self.eof = True break # eof, id not found, will return None @@ -679,7 +683,7 @@ class fastqNamedReader( object ): if not eof: offset = self.file.tell() try: - self.reader.next() + next(self.reader) except StopIteration: eof = True self.file.seek( offset ) diff --git a/lib/tool_shed/util/container_util.py b/lib/tool_shed/util/container_util.py index 670d0db0340..1f4ea75f479 100644 --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import logging from tool_shed.util import common_util @@ -62,8 +64,8 @@ def print_folders( pad, folder ): pad_str = '' for i in range( 1, pad ): pad_str += ' ' - print '%sid: %s key: %s' % ( pad_str, str( folder.id ), folder.key ) + print('%sid: %s key: %s' % ( pad_str, str( folder.id ), folder.key )) for repository_dependency in folder.repository_dependencies: - print ' %s%s' % ( pad_str, repository_dependency.listify ) + print(' %s%s' % ( pad_str, repository_dependency.listify )) for sub_folder in folder.folders: print_folders( pad + 5, sub_folder ) diff --git a/scripts/cleanup_datasets/admin_cleanup_datasets.py b/scripts/cleanup_datasets/admin_cleanup_datasets.py index 983bfe0d9cc..4033f2caded 100755 --- a/scripts/cleanup_datasets/admin_cleanup_datasets.py +++ b/scripts/cleanup_datasets/admin_cleanup_datasets.py @@ -36,9 +36,10 @@ Email Template Variables: Author: Lance Parsons (lparsons@princeton.edu) """ -import ConfigParser -import os +from __future__ import print_function + import logging +import os import shutil import sys import time @@ -47,16 +48,17 @@ from datetime import datetime, timedelta from optparse import OptionParser from time import strftime -from cleanup_datasets import CleanupDatasetsApplication - import sqlalchemy as sa -from sqlalchemy import and_, false from mako.template import Template +from six.moves import configparser +from sqlalchemy import and_, false import galaxy.config import galaxy.model.mapping import galaxy.util +from cleanup_datasets import CleanupDatasetsApplication + log = logging.getLogger() log.setLevel(10) log.addHandler(logging.StreamHandler(sys.stdout)) @@ -102,7 +104,7 @@ def main(): sys.exit() ini_file = args[0] - config_parser = ConfigParser.ConfigParser({'here': os.getcwd()}) + config_parser = configparser.ConfigParser({'here': os.getcwd()}) config_parser.read(ini_file) config_dict = {} for key, value in config_parser.items("app:main"): @@ -130,7 +132,7 @@ def main(): if os.path.exists(default_template): template_file = default_template elif os.path.exists(sample_template_file): - print "Copying %s to %s" % (sample_template_file, default_template) + print("Copying %s to %s" % (sample_template_file, default_template)) shutil.copyfile(sample_template_file, default_template) template_file = default_template else: @@ -147,13 +149,13 @@ def main(): cutoff_time = datetime.utcnow() - timedelta(days=options.days) now = strftime("%Y-%m-%d %H:%M:%S") - print "##########################################" - print "\n# %s - Handling stuff older than %i days" % (now, options.days) + print("##########################################") + print("\n# %s - Handling stuff older than %i days" % (now, options.days)) if options.info_only: - print "# Displaying info only ( --info_only )\n" + print("# Displaying info only ( --info_only )\n") elif options.email_only: - print "# Sending emails only, not deleting ( --email_only )\n" + print("# Sending emails only, not deleting ( --email_only )\n") administrative_delete_datasets( app, cutoff_time, options.days, tool_id=options.tool_id, @@ -225,33 +227,33 @@ def administrative_delete_datasets(app, cutoff_time, cutoff_days, # Mark the HistoryDatasetAssociation as deleted hda.deleted = True app.sa_session.add(hda) - print ("Marked HistoryDatasetAssociation id %d as " - "deleted" % hda.id) + print("Marked HistoryDatasetAssociation id %d as " + "deleted" % hda.id) app.sa_session.flush() emailtemplate = Template(filename=template_file) - for (email, dataset_list) in user_notifications.iteritems(): + for (email, dataset_list) in user_notifications.items(): msgtext = emailtemplate.render(email=email, datasets=dataset_list, cutoff=cutoff_days) subject = "Galaxy Server Cleanup " \ "- %d datasets DELETED" % len(dataset_list) fromaddr = config.email_from - print "" - print "From: %s" % fromaddr - print "To: %s" % email - print "Subject: %s" % subject - print "----------" - print msgtext + print() + print("From: %s" % fromaddr) + print("To: %s" % email) + print("Subject: %s" % subject) + print("----------") + print(msgtext) if not info_only: galaxy.util.send_mail(fromaddr, email, subject, msgtext, config) stop = time.time() - print "" - print "Marked %d dataset instances as deleted" % deleted_instance_count - print "Total elapsed time: ", stop - start - print "##########################################" + print() + print("Marked %d dataset instances as deleted" % deleted_instance_count) + print("Total elapsed time: ", stop - start) + print("##########################################") def _get_tool_id_for_hda(app, hda_id): diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 5811c5be572..7653db51a10 100755 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -import ConfigParser +from __future__ import print_function + import logging import os import shutil @@ -10,6 +11,7 @@ from optparse import OptionParser from time import strftime import sqlalchemy as sa +from six.moves import configparser from sqlalchemy import and_, false, null, true from sqlalchemy.orm import eagerload @@ -19,6 +21,7 @@ import galaxy.config import galaxy.model.mapping from galaxy.exceptions import ObjectNotFound from galaxy.objectstore import build_object_store_from_config +from galaxy.util import unicodify log = logging.getLogger() log.setLevel( 10 ) @@ -93,7 +96,7 @@ def main(): if options.remove_from_disk and options.info_only: parser.error( "remove_from_disk and info_only are mutually exclusive" ) - config_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} ) + config_parser = configparser.ConfigParser( {'here': os.getcwd()} ) config_parser.read( ini_file ) config_dict = {} for key, value in config_parser.items( "app:main" ): @@ -105,15 +108,15 @@ def main(): cutoff_time = datetime.utcnow() - timedelta( days=options.days ) now = strftime( "%Y-%m-%d %H:%M:%S" ) - print "##########################################" - print "\n# %s - Handling stuff older than %i days" % ( now, options.days ) + print("##########################################") + print("\n# %s - Handling stuff older than %i days" % ( now, options.days )) if options.info_only: - print "# Displaying info only ( --info_only )\n" + print("# Displaying info only ( --info_only )\n") elif options.remove_from_disk: - print "Datasets will be removed from disk.\n" + print("Datasets will be removed from disk.\n") else: - print "Datasets will NOT be removed from disk.\n" + print("Datasets will NOT be removed from disk.\n") if options.delete_userless_histories: delete_userless_histories( app, cutoff_time, info_only=options.info_only, force_retry=options.force_retry ) @@ -149,15 +152,15 @@ def delete_userless_histories( app, cutoff_time, info_only=False, force_retry=Fa app.model.History.table.c.update_time < cutoff_time ) ) for history in histories: if not info_only: - print "Deleting history id ", history.id + print("Deleting history id ", history.id) history.deleted = True app.sa_session.add( history ) app.sa_session.flush() history_count += 1 stop = time.time() - print "Deleted %d histories" % history_count - print "Elapsed time: ", stop - start - print "##########################################" + print("Deleted %d histories" % history_count) + print("Elapsed time: ", stop - start) + print("##########################################") def purge_histories( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ): @@ -180,7 +183,7 @@ def purge_histories( app, cutoff_time, remove_from_disk, info_only=False, force_ app.model.History.table.c.update_time < cutoff_time ) ) \ .options( eagerload( 'datasets' ) ) for history in histories: - print ("### Processing history id %d (%s)" % (history.id, history.name)).encode('utf-8') + print("### Processing history id %d (%s)" % (history.id, unicodify(history.name))) for dataset_assoc in history.datasets: _purge_dataset_instance( dataset_assoc, app, remove_from_disk, info_only=info_only ) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable if not info_only: @@ -189,17 +192,17 @@ def purge_histories( app, cutoff_time, remove_from_disk, info_only=False, force_ # if we should ever delete info like this from the db though, so commented out for now... # for dhp in history.default_permissions: # dhp.delete() - print "Purging history id ", history.id + print("Purging history id ", history.id) history.purged = True app.sa_session.add( history ) app.sa_session.flush() else: - print "History id %d will be purged (without 'info_only' mode)" % history.id + print("History id %d will be purged (without 'info_only' mode)" % history.id) history_count += 1 stop = time.time() - print 'Purged %d histories.' % history_count - print "Elapsed time: ", stop - start - print "##########################################" + print('Purged %d histories.' % history_count) + print("Elapsed time: ", stop - start) + print("##########################################") def purge_libraries( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ): @@ -222,15 +225,15 @@ def purge_libraries( app, cutoff_time, remove_from_disk, info_only=False, force_ for library in libraries: _purge_folder( library.root_folder, app, remove_from_disk, info_only=info_only ) if not info_only: - print "Purging library id ", library.id + print("Purging library id ", library.id) library.purged = True app.sa_session.add( library ) app.sa_session.flush() library_count += 1 stop = time.time() - print '# Purged %d libraries .' % library_count - print "Elapsed time: ", stop - start - print "##########################################" + print('# Purged %d libraries .' % library_count) + print("Elapsed time: ", stop - start) + print("##########################################") def purge_folders( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ): @@ -254,9 +257,9 @@ def purge_folders( app, cutoff_time, remove_from_disk, info_only=False, force_re _purge_folder( folder, app, remove_from_disk, info_only=info_only ) folder_count += 1 stop = time.time() - print '# Purged %d folders.' % folder_count - print "Elapsed time: ", stop - start - print "##########################################" + print('# Purged %d folders.' % folder_count) + print("Elapsed time: ", stop - start) + print("##########################################") def delete_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ): @@ -301,7 +304,7 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_ library_dataset_ids = [ row.id for row in library_dataset_ids_query.execute() ] dataset_ids = [] for library_dataset_id in library_dataset_ids: - print "######### Processing LibraryDataset id:", library_dataset_id + print("######### Processing LibraryDataset id:", library_dataset_id) # Get the LibraryDataset and the current LibraryDatasetDatasetAssociation objects ld = app.sa_session.query( app.model.LibraryDataset ).get( library_dataset_id ) ldda = ld.library_dataset_dataset_association @@ -311,16 +314,16 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_ if not ldda.deleted: ldda.deleted = True app.sa_session.add( ldda ) - print "Marked associated LibraryDatasetDatasetAssociation id %d as deleted" % ldda.id + print("Marked associated LibraryDatasetDatasetAssociation id %d as deleted" % ldda.id) for expired_ldda in ld.expired_datasets: if not expired_ldda.deleted: expired_ldda.deleted = True app.sa_session.add( expired_ldda ) - print "Marked associated expired LibraryDatasetDatasetAssociation id %d as deleted" % ldda.id + print("Marked associated expired LibraryDatasetDatasetAssociation id %d as deleted" % ldda.id) # Mark the LibraryDataset as purged ld.purged = True app.sa_session.add( ld ) - print "Marked LibraryDataset id %d as purged" % ld.id + print("Marked LibraryDataset id %d as purged" % ld.id) app.sa_session.flush() # Add all datasets associated with Histories to our list dataset_ids.extend( [ row.id for row in history_dataset_ids_query.execute() ] ) @@ -330,9 +333,9 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_ if dataset.id in skip: continue skip.append( dataset.id ) - print "######### Processing dataset id:", dataset_id + print("######### Processing dataset id:", dataset_id) if not _dataset_is_deletable( dataset ): - print "Dataset is not deletable (shared between multiple histories/libraries, at least one is not deleted)" + print("Dataset is not deletable (shared between multiple histories/libraries, at least one is not deleted)") continue deleted_dataset_count += 1 for dataset_instance in dataset.history_associations + dataset.library_associations: @@ -340,9 +343,9 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_ _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_children=True, info_only=info_only, is_deletable=True ) deleted_instance_count += 1 stop = time.time() - print "Examined %d datasets, marked %d datasets and %d dataset instances (HDA) as deleted" % ( len( skip ), deleted_dataset_count, deleted_instance_count ) - print "Total elapsed time: ", stop - start - print "##########################################" + print("Examined %d datasets, marked %d datasets and %d dataset instances (HDA) as deleted" % ( len( skip ), deleted_dataset_count, deleted_instance_count )) + print("Total elapsed time: ", stop - start) + print("##########################################") def purge_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ): @@ -371,35 +374,35 @@ def purge_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_r except: pass stop = time.time() - print 'Purged %d datasets' % dataset_count + print('Purged %d datasets' % dataset_count) if remove_from_disk: - print 'Freed disk space: ', disk_space - print "Elapsed time: ", stop - start - print "##########################################" + print('Freed disk space: ', disk_space) + print("Elapsed time: ", stop - start) + print("##########################################") def _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_children=True, info_only=False, is_deletable=False ): # A dataset_instance is either a HDA or an LDDA. Purging a dataset instance marks the instance as deleted, # and marks the associated dataset as deleted if it is not associated with another active DatsetInstance. if not info_only: - print "Marking as deleted: %s id %d (for dataset id %d)" % \ - ( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id ) + print("Marking as deleted: %s id %d (for dataset id %d)" % + ( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id )) dataset_instance.mark_deleted( include_children=include_children ) dataset_instance.clear_associated_files() app.sa_session.add( dataset_instance ) app.sa_session.flush() app.sa_session.refresh( dataset_instance.dataset ) else: - print "%s id %d (for dataset id %d) will be marked as deleted (without 'info_only' mode)" % \ - ( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id ) + print("%s id %d (for dataset id %d) will be marked as deleted (without 'info_only' mode)" % + ( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id )) if is_deletable or _dataset_is_deletable( dataset_instance.dataset ): # Calling methods may have already checked _dataset_is_deletable, if so, is_deletable should be True _delete_dataset( dataset_instance.dataset, app, remove_from_disk, info_only=info_only, is_deletable=is_deletable ) else: if info_only: - print "Not deleting dataset ", dataset_instance.dataset.id, " (will be possibly deleted without 'info_only' mode)" + print("Not deleting dataset ", dataset_instance.dataset.id, " (will be possibly deleted without 'info_only' mode)") else: - print "Not deleting dataset %d (shared between multiple histories/libraries, at least one not deleted)" % dataset_instance.dataset.id + print("Not deleting dataset %d (shared between multiple histories/libraries, at least one not deleted)" % dataset_instance.dataset.id) # need to purge children here if include_children: for child in dataset_instance.children: @@ -415,7 +418,7 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab # Marks a base dataset as deleted, hdas/lddas associated with dataset can no longer be undeleted. # Metadata files attached to associated dataset Instances is removed now. if not is_deletable and not _dataset_is_deletable( dataset ): - print "This Dataset (%i) is not deletable, associated Metadata Files will not be removed.\n" % ( dataset.id ) + print("This Dataset (%i) is not deletable, associated Metadata Files will not be removed.\n" % ( dataset.id )) else: # Mark all associated MetadataFiles as deleted and purged and remove them from disk metadata_files = [] @@ -433,29 +436,29 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab if remove_from_disk: op_description = op_description + " and purged from disk" if info_only: - print "The following metadata files attached to associations of Dataset '%s' will be %s (without 'info_only' mode):" % ( dataset.id, op_description ) + print("The following metadata files attached to associations of Dataset '%s' will be %s (without 'info_only' mode):" % ( dataset.id, op_description )) else: - print "The following metadata files attached to associations of Dataset '%s' have been %s:" % ( dataset.id, op_description ) + print("The following metadata files attached to associations of Dataset '%s' have been %s:" % ( dataset.id, op_description )) if remove_from_disk: try: - print "Removing disk file ", metadata_file.file_name + print("Removing disk file ", metadata_file.file_name) os.unlink( metadata_file.file_name ) except Exception as e: - print "Error, exception: %s caught attempting to purge metadata file %s\n" % ( str( e ), metadata_file.file_name ) + print("Error, exception: %s caught attempting to purge metadata file %s\n" % ( str( e ), metadata_file.file_name )) metadata_file.purged = True app.sa_session.add( metadata_file ) app.sa_session.flush() metadata_file.deleted = True app.sa_session.add( metadata_file ) app.sa_session.flush() - print "%s" % metadata_file.file_name + print("%s" % metadata_file.file_name) if not info_only: - print "Deleting dataset id", dataset.id + print("Deleting dataset id", dataset.id) dataset.deleted = True app.sa_session.add( dataset ) app.sa_session.flush() else: - print "Dataset %i will be deleted (without 'info_only' mode)" % ( dataset.id ) + print("Dataset %i will be deleted (without 'info_only' mode)" % ( dataset.id )) def _purge_dataset( app, dataset, remove_from_disk, info_only=False ): @@ -466,7 +469,7 @@ def _purge_dataset( app, dataset, remove_from_disk, info_only=False ): # Remove files from disk and update the database if remove_from_disk: # TODO: should permissions on the dataset be deleted here? - print "Removing disk, file ", dataset.file_name + print("Removing disk, file ", dataset.file_name) os.unlink( dataset.file_name ) # Remove associated extra files from disk if they exist if dataset.extra_files_path and os.path.exists( dataset.extra_files_path ): @@ -480,32 +483,32 @@ def _purge_dataset( app, dataset, remove_from_disk, info_only=False ): for user in usage_users: user.adjust_total_disk_usage(-dataset.get_total_size()) app.sa_session.add( user ) - print "Purging dataset id", dataset.id + print("Purging dataset id", dataset.id) dataset.purged = True app.sa_session.add( dataset ) app.sa_session.flush() else: - print "Dataset %i will be purged (without 'info_only' mode)" % (dataset.id) + print("Dataset %i will be purged (without 'info_only' mode)" % (dataset.id)) else: - print "This dataset (%i) is not purgable, the file (%s) will not be removed.\n" % ( dataset.id, dataset.file_name ) + print("This dataset (%i) is not purgable, the file (%s) will not be removed.\n" % ( dataset.id, dataset.file_name )) except OSError as exc: - print "Error, dataset file has already been removed: %s" % str( exc ) - print "Purging dataset id", dataset.id + print("Error, dataset file has already been removed: %s" % str( exc )) + print("Purging dataset id", dataset.id) dataset.purged = True app.sa_session.add( dataset ) app.sa_session.flush() except ObjectNotFound: - print "Dataset %i cannot be found in the object store" % dataset.id + print("Dataset %i cannot be found in the object store" % dataset.id) except Exception as exc: - print "Error attempting to purge data file: ", dataset.file_name, " error: ", str( exc ) + print("Error attempting to purge data file: ", dataset.file_name, " error: ", str( exc )) else: - print "Error: '%s' has not previously been deleted, so it cannot be purged\n" % dataset.file_name + print("Error: '%s' has not previously been deleted, so it cannot be purged\n" % dataset.file_name) def _purge_folder( folder, app, remove_from_disk, info_only=False ): """Purges a folder and its contents, recursively""" for ld in folder.datasets: - print "Deleting library dataset id ", ld.id + print("Deleting library dataset id ", ld.id) ld.deleted = True for ldda in [ld.library_dataset_dataset_association] + ld.expired_datasets: _purge_dataset_instance( ldda, app, remove_from_disk, info_only=info_only ) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable @@ -513,7 +516,7 @@ def _purge_folder( folder, app, remove_from_disk, info_only=False ): _purge_folder( sub_folder, app, remove_from_disk, info_only=info_only ) if not info_only: # TODO: should the folder permissions be deleted here? - print "Purging folder id ", folder.id + print("Purging folder id ", folder.id) folder.purged = True app.sa_session.add( folder ) app.sa_session.flush() diff --git a/test/api/test_workflows_from_yaml.py b/test/api/test_workflows_from_yaml.py index f6356599eef..cc269146a54 100644 --- a/test/api/test_workflows_from_yaml.py +++ b/test/api/test_workflows_from_yaml.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os from .test_workflows import BaseWorkflowsApiTestCase @@ -241,7 +243,7 @@ steps: input1: $link: the_pause """) - print self._get("workflows/%s/download" % workflow_id).json() + print(self._get("workflows/%s/download" % workflow_id).json()) def test_implicit_connections( self ): workflow_id = self._upload_yaml_workflow(""" @@ -273,7 +275,7 @@ steps: $link: test_input """) workflow = self._get("workflows/%s/download" % workflow_id).json() - print workflow + print(workflow) def _steps_by_label(self, workflow_as_dict): by_label = {} diff --git a/test/base/interactor.py b/test/base/interactor.py index 3fbc829c4aa..69e32f1b1a0 100644 --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -1,10 +1,12 @@ +from __future__ import print_function + import os import re from json import dumps from logging import getLogger -from StringIO import StringIO from requests import get, post, delete, patch +from six import StringIO from galaxy import util from galaxy.tools.parser.interface import TestCollectionDef @@ -64,7 +66,7 @@ class GalaxyInteractorApi( object ): job_id = self._dataset_provenance( history_id, hid )[ "job_id" ] outputs = self._get( "jobs/%s/outputs" % ( job_id ) ).json() - for designation, ( primary_outfile, primary_attributes ) in primary_datasets.iteritems(): + for designation, ( primary_outfile, primary_attributes ) in primary_datasets.items(): primary_output = None for output in outputs: if output[ "name" ] == '__new_primary_file_%s|%s__' % ( name, designation ): @@ -90,7 +92,7 @@ class GalaxyInteractorApi( object ): def _verify_metadata( self, history_id, hid, attributes ): metadata = attributes.get( 'metadata', {} ).copy() - for key, value in metadata.copy().iteritems(): + for key, value in metadata.copy().items(): new_key = "metadata_%s" % key metadata[ new_key ] = metadata[ key ] del metadata[ key ] @@ -100,7 +102,7 @@ class GalaxyInteractorApi( object ): if metadata: dataset = self._get( "histories/%s/contents/%s" % ( history_id, hid ) ).json() - for key, value in metadata.iteritems(): + for key, value in metadata.items(): try: dataset_value = dataset.get( key, None ) if dataset_value != value: @@ -189,7 +191,7 @@ class GalaxyInteractorApi( object ): # tool will have uncompressed it on the fly. inputs_tree = testdef.inputs.copy() - for key, value in inputs_tree.iteritems(): + for key, value in inputs_tree.items(): values = [value] if not isinstance(value, list) else value new_values = [] for value in values: @@ -203,7 +205,7 @@ class GalaxyInteractorApi( object ): inputs_tree[ key ] = new_values # HACK: Flatten single-value lists. Required when using expand_grouping - for key, value in inputs_tree.iteritems(): + for key, value in inputs_tree.items(): if isinstance(value, list) and len(value) == 1: inputs_tree[key] = value[0] @@ -306,11 +308,11 @@ class GalaxyInteractorApi( object ): def _summarize_history_errors( self, history_id ): if history_id is None: raise ValueError("_summarize_history_errors passed empty history_id") - print "History with id %s in error - summary of datasets in error below." % history_id + print("History with id %s in error - summary of datasets in error below." % history_id) try: history_contents = self.__contents( history_id ) except Exception: - print "*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*" + print("*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*") for history_content in history_contents: if history_content[ 'history_content_type'] != 'dataset': @@ -320,27 +322,27 @@ class GalaxyInteractorApi( object ): if dataset[ 'state' ] != 'error': continue - print ERROR_MESSAGE_DATASET_SEP + print(ERROR_MESSAGE_DATASET_SEP) dataset_id = dataset.get( 'id', None ) - print "| %d - %s (HID - NAME) " % ( int( dataset['hid'] ), dataset['name'] ) + print("| %d - %s (HID - NAME) " % ( int( dataset['hid'] ), dataset['name'] )) try: dataset_info = self._dataset_info( history_id, dataset_id ) - print "| Dataset Blurb:" - print self.format_for_error( dataset_info.get( "misc_blurb", "" ), "Dataset blurb was empty." ) - print "| Dataset Info:" - print self.format_for_error( dataset_info.get( "misc_info", "" ), "Dataset info is empty." ) + print("| Dataset Blurb:") + print(self.format_for_error( dataset_info.get( "misc_blurb", "" ), "Dataset blurb was empty." )) + print("| Dataset Info:") + print(self.format_for_error( dataset_info.get( "misc_info", "" ), "Dataset info is empty." )) except Exception: - print "| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*" + print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*") try: provenance_info = self._dataset_provenance( history_id, dataset_id ) - print "| Dataset Job Standard Output:" - print self.format_for_error( provenance_info.get( "stdout", "" ), "Standard output was empty." ) - print "| Dataset Job Standard Error:" - print self.format_for_error( provenance_info.get( "stderr", "" ), "Standard error was empty." ) + print("| Dataset Job Standard Output:") + print(self.format_for_error( provenance_info.get( "stdout", "" ), "Standard output was empty." )) + print("| Dataset Job Standard Error:") + print(self.format_for_error( provenance_info.get( "stderr", "" ), "Standard error was empty." )) except Exception: - print "| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*" - print "|" - print ERROR_MESSAGE_DATASET_SEP + print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*") + print("|") + print(ERROR_MESSAGE_DATASET_SEP) def format_for_error( self, blob, empty_message, prefix="| " ): contents = "\n".join([ "%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r") ] ) diff --git a/test/base/twilltestcase.py b/test/base/twilltestcase.py index c42255396b6..9e4bef3f8a7 100644 --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1,21 +1,22 @@ +from __future__ import print_function + import logging import os import pprint import shutil -import StringIO import tarfile import tempfile import time import unittest -import urllib import zipfile from json import loads -from urlparse import urlparse from xml.etree import ElementTree -from markupsafe import escape import twill import twill.commands as tc +from markupsafe import escape +from six import string_types, StringIO +from six.moves.urllib.parse import unquote, urlencode, urlparse from twill.other_packages._mechanize_dist import ClientForm from galaxy.web import security @@ -31,7 +32,7 @@ from galaxy.tools.verify import ( # Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? -buffer = StringIO.StringIO() +buffer = StringIO() twill.set_output( buffer ) tc.config( 'use_tidy', 0 ) @@ -422,7 +423,7 @@ class TwillTestCase( unittest.TestCase ): try: json_data = self.get_history_from_api( show_deleted=show_deleted, show_details=True ) check_result = check_fn( json_data ) - assert check_result, 'failed check_fn: %s (got %s)' % ( check_fn.func_name, str( check_result ) ) + assert check_result, 'failed check_fn: %s (got %s)' % ( check_fn.__name__, str( check_result ) ) except Exception as e: log.exception( e ) log.debug( 'json_data: %s', ( '\n' + pprint.pformat( json_data ) if json_data else '(no match)' ) ) @@ -1616,7 +1617,7 @@ class TwillTestCase( unittest.TestCase ): def rename_history( self, id, old_name, new_name ): """Rename an existing history""" self.visit_url( "/history/rename", params=dict( id=id, name=new_name ) ) - check_str = 'History: %s renamed to: %s' % ( old_name, urllib.unquote( new_name ) ) + check_str = 'History: %s renamed to: %s' % ( old_name, unquote( new_name ) ) self.check_page_for_string( check_str ) def rename_sample_datasets( self, sample_id, sample_dataset_ids, new_sample_dataset_names, strings_displayed=[], strings_displayed_after_submit=[] ): @@ -1748,7 +1749,7 @@ class TwillTestCase( unittest.TestCase ): if i == form_no: break # To help with debugging a tool, print out the form controls when the test fails - print "form '%s' contains the following controls ( note the values )" % f.name + print("form '%s' contains the following controls ( note the values )" % f.name) controls = {} formcontrols = self.get_form_controls( f ) hc_prefix = '= ( 2, 4 ) @@ -14,12 +16,12 @@ out_name = sys.argv[2] out = open(out_name, 'wt') try: - page = urllib.urlopen(url) + page = urlopen(url) while 1: data = page.read(BUFFER) if not data: break out.write(data) except Exception as e: - print 'Error getting the data -> %s' % e + print('Error getting the data -> %s' % e) out.close() diff --git a/tools/data_source/genbank.py b/tools/data_source/genbank.py index 74ffd10aab5..1276814e22b 100644 --- a/tools/data_source/genbank.py +++ b/tools/data_source/genbank.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function + import sys import textwrap @@ -21,12 +23,12 @@ if __name__ == '__main__': text = sys.argv[2] output_file = sys.argv[3] - print 'Searching for %s
' % text + print('Searching for %s
' % text) # check if inputs are all numbers try: gi_list = text.split() - tmp = map(int, gi_list) + [int(_) for _ in gi_list] except ValueError: gi_list = GenBank.search_for(text, max_ids=10) @@ -37,5 +39,5 @@ if __name__ == '__main__': res = ncbi_dict[gid] head, body = make_fasta(res) fp.write(head + body + '\n') - print head + print(head) fp.close() diff --git a/tools/data_source/hbvar_filter.py b/tools/data_source/hbvar_filter.py index 688c69f59c0..a30b6450c0c 100644 --- a/tools/data_source/hbvar_filter.py +++ b/tools/data_source/hbvar_filter.py @@ -1,7 +1,8 @@ # TODO: Set dbkey to proper UCSC build, if known import shutil import tempfile -import urllib + +from six.moves.urllib.request import urlopen from galaxy import datatypes @@ -12,7 +13,7 @@ def exec_before_job( app, inp_data, out_data, param_dict, tool=None): data_type = param_dict.get( 'type', 'txt' ) if data_type == 'txt': data_type = 'interval' # All data is TSV, assume interval - name, data = out_data.items()[0] + name, data = list(out_data.items())[0] data = app.datatypes_registry.change_datatype(data, data_type) data.name = data_name out_data[name] = data @@ -30,11 +31,11 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No MAX_SIZE = CHUNK_SIZE * 100 try: - page = urllib.urlopen(URL) + page = urlopen(URL) except Exception as exc: raise Exception('Problems connecting to %s (%s)' % (URL, exc) ) - name, data = out_data.items()[0] + name, data = list(out_data.items())[0] fp = open(data.file_name, 'wb') size = 0 diff --git a/tools/data_source/import.py b/tools/data_source/import.py index ac9c815005c..8c723c97d5d 100644 --- a/tools/data_source/import.py +++ b/tools/data_source/import.py @@ -1,9 +1,10 @@ #!/usr/bin/env python - """ Script that imports locally stored data as a new dataset for the user Usage: import id outputfile """ +from __future__ import print_function + import os import sys @@ -35,10 +36,10 @@ id2name = { fname = id2name.get(dataid, '') if not fname: - print 'Importing invalid data %s' % dataid + print('Importing invalid data %s' % dataid) sys.exit() else: - print 'Imported %s' % fname + print('Imported %s' % fname) # this path is hardcoded inp_name = os.path.join('database', 'import', fname) @@ -46,7 +47,7 @@ inp_name = os.path.join('database', 'import', fname) try: inp = open(inp_name, 'rt') except: - print 'Could not find file %s' % inp_name + print('Could not find file %s' % inp_name) sys.exit() out = open(out_name, 'wt') diff --git a/tools/data_source/microbial_import.py b/tools/data_source/microbial_import.py index d3fbf63493b..9dfcb916ea8 100644 --- a/tools/data_source/microbial_import.py +++ b/tools/data_source/microbial_import.py @@ -1,9 +1,10 @@ #!/usr/bin/env python - """ Script that imports locally stored data as a new dataset for the user Usage: import id outputfile """ +from __future__ import print_function + import sys from shutil import copyfile @@ -52,7 +53,7 @@ try: available_files[uid] = (description, path, build, file_type, chr_acc) except: - print >>sys.stderr, "It appears that the configuration file for this tool is missing." + print("It appears that the configuration file for this tool is missing.", file=sys.stderr) # create list of tuples of (displayName,FileName,build) for desired files desired_files = [] @@ -68,18 +69,18 @@ while not file1_copied: try: first_file = desired_files.pop(0) except: - print >>sys.stderr, "There were no valid files requested." + print("There were no valid files requested.", file=sys.stderr) sys.exit() file1_desc, file1_path, file1_build, file1_type, file1_chr_acc = first_file try: copyfile(file1_path, out_file1) - print "#File1\t" + file1_desc + "\t" + file1_chr_acc + "\t" + file1_build + "\t" + file1_type + print("#File1\t" + file1_desc + "\t" + file1_chr_acc + "\t" + file1_build + "\t" + file1_type) file1_copied = True except: - print >>sys.stderr, "The file specified is missing." + print("The file specified is missing.", file=sys.stderr) continue # Tell post-process filter where remaining files reside for extra_output in desired_files: file_desc, file_path, file_build, file_type, file_chr_acc = extra_output - print "#NewFile\t" + file_desc + "\t" + file_chr_acc + "\t" + file_build + "\t" + file_path + "\t" + file_type + print("#NewFile\t" + file_desc + "\t" + file_chr_acc + "\t" + file_build + "\t" + file_path + "\t" + file_type) diff --git a/tools/data_source/microbial_import_code.py b/tools/data_source/microbial_import_code.py index 638c3e52627..871ca1ef887 100644 --- a/tools/data_source/microbial_import_code.py +++ b/tools/data_source/microbial_import_code.py @@ -1,3 +1,5 @@ +from __future__ import print_function + from shutil import copyfile from galaxy import tools @@ -88,17 +90,17 @@ def load_microbial_data( GALAXY_DATA_INDEX_DIR, sep='\t' ): # post processing, set build for data and add additional data to history def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr): - base_dataset = out_data.items()[0][1] + base_dataset = list(out_data.items())[0][1] history = base_dataset.history if history is None: - print "unknown history!" + print("unknown history!") return kingdom = param_dict.get( 'kingdom', None ) org = param_dict.get( 'org', None ) # if not (kingdom or group or org): if not (kingdom or org): - print "Parameters are not available." + print("Parameters are not available.") # workflow passes galaxy.tools.parameters.basic.UnvalidatedValue instead of values if isinstance( kingdom, tools.parameters.basic.UnvalidatedValue ): kingdom = kingdom.value @@ -116,7 +118,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr chr = fields[2] dbkey = fields[3] file_type = fields[4] - name, data = out_data.items()[0] + name, data = list(out_data.items())[0] data.set_size() basic_name = data.name data.name = data.name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] + " for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")" diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 0450538f9fc..a123aa327f2 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -3,6 +3,7 @@ # WARNING: Changes in this tool (particularly as related to parsing) may need # to be reflected in galaxy.web.controllers.tool_runner and galaxy.tools +from __future__ import print_function import codecs import gzip @@ -10,10 +11,11 @@ import os import shutil import sys import tempfile -import urllib import zipfile from json import dumps, loads +from six.moves.urllib.request import urlopen + from galaxy import util from galaxy.datatypes import sniff from galaxy.datatypes.binary import Binary @@ -63,7 +65,7 @@ def safe_dict(d): http://mellowmachines.com/blog/2009/06/exploding-dictionary-with-unicode-keys-as-python-arguments/ """ if isinstance(d, dict): - return dict([(k.encode('utf-8'), safe_dict(v)) for k, v in d.iteritems()]) + return dict([(k.encode('utf-8'), safe_dict(v)) for k, v in d.items()]) elif isinstance(d, list): return [safe_dict(x) for x in d] else: @@ -94,7 +96,7 @@ def add_file( dataset, registry, json_file, output_path ): if dataset.type == 'url': try: - page = urllib.urlopen( dataset.path ) # page will be .close()ed by sniff methods + page = urlopen( dataset.path ) # page will be .close()ed by sniff methods temp_name, dataset.is_multi_byte = sniff.stream_to_file( page, prefix='url_paste', source_encoding=util.get_charset_from_http_headers( page.headers ) ) except Exception as e: file_err( 'Unable to fetch %s\n%s' % ( dataset.path, str( e ) ), dataset, json_file ) @@ -168,7 +170,7 @@ def add_file( dataset, registry, json_file, output_path ): dataset.path = uncompressed else: shutil.move( uncompressed, dataset.path ) - os.chmod(dataset.path, 0644) + os.chmod(dataset.path, 0o644) dataset.name = dataset.name.rstrip( '.gz' ) data_type = 'gzip' if not data_type and bz2 is not None: @@ -201,7 +203,7 @@ def add_file( dataset, registry, json_file, output_path ): dataset.path = uncompressed else: shutil.move( uncompressed, dataset.path ) - os.chmod(dataset.path, 0644) + os.chmod(dataset.path, 0o644) dataset.name = dataset.name.rstrip( '.bz2' ) data_type = 'bz2' if not data_type: @@ -258,7 +260,7 @@ def add_file( dataset, registry, json_file, output_path ): dataset.path = uncompressed else: shutil.move( uncompressed, dataset.path ) - os.chmod(dataset.path, 0644) + os.chmod(dataset.path, 0o644) dataset.name = uncompressed_name data_type = 'zip' if not data_type: @@ -353,7 +355,7 @@ def add_file( dataset, registry, json_file, output_path ): def add_composite_file( dataset, json_file, output_path, files_path ): if dataset.composite_files: os.mkdir( files_path ) - for name, value in dataset.composite_files.iteritems(): + for name, value in dataset.composite_files.items(): value = util.bunch.Bunch( **value ) if dataset.composite_file_paths[ value.name ] is None and not value.optional: file_err( 'A required composite data file was not provided (%s)' % name, dataset, json_file ) @@ -363,7 +365,7 @@ def add_composite_file( dataset, json_file, output_path, files_path ): isurl = dp.find('://') != -1 # todo fixme if isurl: try: - temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dp ), prefix='url_paste' ) + temp_name, dataset.is_multi_byte = sniff.stream_to_file( urlopen( dp ), prefix='url_paste' ) except Exception as e: file_err( 'Unable to fetch %s\n%s' % ( dp, str( e ) ), dataset, json_file ) return @@ -397,7 +399,7 @@ def output_adjacent_tmpdir( output_path ): def __main__(): if len( sys.argv ) < 4: - print >>sys.stderr, 'usage: upload.py ...' + print('usage: upload.py ...', file=sys.stderr) sys.exit( 1 ) output_paths = parse_outputs( sys.argv[4:] ) @@ -412,7 +414,7 @@ def __main__(): try: output_path = output_paths[int( dataset.dataset_id )][0] except: - print >>sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id + print('Output path for dataset %s not found on command line' % dataset.dataset_id, file=sys.stderr) sys.exit( 1 ) if dataset.type == 'composite': files_path = output_paths[int( dataset.dataset_id )][1]