From 6f98634dd51ac40848a2e16662782378f8022bfd Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 27 Jun 2017 03:42:44 +0200 Subject: [PATCH] Fix import order and Python3 compatibility for 51 files xref. https://github.com/galaxyproject/galaxy/issues/1715 Also remove unused `get_track_window()` method of some datatypes. --- .ci/flake8_lint_include_list.txt | 93 +++---------------- .ci/py3_sources.txt | 31 +------ lib/galaxy/dataset_collections/matching.py | 14 ++- lib/galaxy/dataset_collections/registry.py | 7 +- lib/galaxy/dataset_collections/structure.py | 2 +- lib/galaxy/dataset_collections/types/list.py | 6 +- .../dataset_collections/types/paired.py | 4 +- lib/galaxy/datatypes/blast.py | 10 +- lib/galaxy/datatypes/chrominfo.py | 2 +- .../bedgraph_to_array_tree_converter.py | 7 +- .../converters/fasta_to_tabular_converter.py | 3 +- .../datatypes/converters/interval_to_fli.py | 2 +- .../converters/pbed_ldreduced_converter.py | 2 +- .../ref_to_seq_taxonomy_converter.py | 3 +- lib/galaxy/datatypes/converters/sam_to_bam.py | 8 +- .../vcf_to_interval_index_converter.py | 4 +- .../wiggle_to_array_tree_converter.py | 2 +- .../converters/wiggle_to_simple_converter.py | 5 +- lib/galaxy/datatypes/coverage.py | 23 ----- lib/galaxy/datatypes/dataproviders/chunk.py | 12 ++- lib/galaxy/datatypes/dataproviders/column.py | 4 +- lib/galaxy/datatypes/dataproviders/dataset.py | 30 +++--- .../datatypes/dataproviders/decorators.py | 11 ++- .../datatypes/dataproviders/external.py | 21 +++-- .../datatypes/dataproviders/hierarchy.py | 9 +- lib/galaxy/datatypes/dataproviders/line.py | 4 +- .../display_applications/application.py | 54 ++++++----- .../display_applications/parameters.py | 12 ++- lib/galaxy/datatypes/graph.py | 14 +-- lib/galaxy/datatypes/interval.py | 35 ------- lib/galaxy/datatypes/molecules.py | 19 ++-- lib/galaxy/datatypes/mothur.py | 7 +- lib/galaxy/datatypes/neo4j.py | 6 +- lib/galaxy/datatypes/qualityscore.py | 5 +- lib/galaxy/datatypes/sniff.py | 12 ++- lib/galaxy/datatypes/triples.py | 13 ++- lib/galaxy/datatypes/util/gff_util.py | 18 ++-- lib/galaxy/datatypes/xml.py | 9 +- lib/galaxy/dependencies/__init__.py | 4 +- lib/galaxy/model/base.py | 11 ++- lib/galaxy/model/custom_types.py | 17 ++-- lib/galaxy/model/search.py | 28 ++++-- .../model/tool_shed_install/__init__.py | 8 +- lib/galaxy/model/tool_shed_install/mapping.py | 26 +++++- .../model/tool_shed_install/migrate/check.py | 19 ++-- lib/galaxy/security/__init__.py | 4 +- lib/galaxy/security/passwords.py | 13 +-- lib/galaxy/webhooks/__init__.py | 6 +- lib/galaxy/work/context.py | 4 +- lib/galaxy/workflow/modules.py | 46 +++++---- lib/galaxy/workflow/render.py | 6 +- lib/galaxy/workflow/run_request.py | 18 ++-- lib/galaxy/workflow/scheduling_manager.py | 14 ++- lib/log_tempfile.py | 3 +- 54 files changed, 350 insertions(+), 400 deletions(-) diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index e81f9131e37..05a81fcdd66 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -9,72 +9,15 @@ lib/galaxy/actions/ lib/galaxy/auth/ lib/galaxy/config.py lib/galaxy/containers/ -lib/galaxy/dataset_collections/builder.py -lib/galaxy/dataset_collections/__init__.py -lib/galaxy/dataset_collections/structure.py -lib/galaxy/dataset_collections/subcollections.py -lib/galaxy/dataset_collections/type_description.py -lib/galaxy/dataset_collections/types/__init__.py -lib/galaxy/datatypes/assembly.py -lib/galaxy/datatypes/binary.py -lib/galaxy/datatypes/checkers.py -lib/galaxy/datatypes/constructive_solid_geometry.py -lib/galaxy/datatypes/converters/bcf_bgzip_to_bcf_converter.py -lib/galaxy/datatypes/converters/bcf_to_bcf_bgzip_converter.py -lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py -lib/galaxy/datatypes/converters/bed_to_gff_converter.py -lib/galaxy/datatypes/converters/bgzip.py -lib/galaxy/datatypes/converters/fasta_to_len.py -lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py -lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py -lib/galaxy/datatypes/converters/fastq_to_fqtoc.py -lib/galaxy/datatypes/converters/gff_to_bed_converter.py -lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py -lib/galaxy/datatypes/converters/__init__.py -lib/galaxy/datatypes/converters/interval_to_bed_converter.py -lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py -lib/galaxy/datatypes/converters/interval_to_coverage.py -lib/galaxy/datatypes/converters/interval_to_fli.py -lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py -lib/galaxy/datatypes/converters/interval_to_tabix_converter.py -lib/galaxy/datatypes/converters/lped_to_fped_converter.py -lib/galaxy/datatypes/converters/lped_to_pbed_converter.py -lib/galaxy/datatypes/converters/maf_to_fasta_converter.py -lib/galaxy/datatypes/converters/maf_to_interval_converter.py -lib/galaxy/datatypes/converters/pbed_to_lped_converter.py -lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py -lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py -lib/galaxy/datatypes/converters/tabular_to_dbnsfp.py -lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py -lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py -lib/galaxy/datatypes/coverage.py -lib/galaxy/datatypes/dataproviders/base.py -lib/galaxy/datatypes/dataproviders/exceptions.py -lib/galaxy/datatypes/dataproviders/__init__.py -lib/galaxy/datatypes/data.py -lib/galaxy/datatypes/display_applications/__init__.py -lib/galaxy/datatypes/display_applications/util.py -lib/galaxy/datatypes/genetics.py -lib/galaxy/datatypes/images.py -lib/galaxy/datatypes/__init__.py -lib/galaxy/datatypes/interval.py -lib/galaxy/datatypes/metadata.py -lib/galaxy/datatypes/msa.py -lib/galaxy/datatypes/ngsindex.py -lib/galaxy/datatypes/proteomics.py -lib/galaxy/datatypes/registry.py -lib/galaxy/datatypes/sequence.py -lib/galaxy/datatypes/tabular.py -lib/galaxy/datatypes/text.py -lib/galaxy/datatypes/tracks.py -lib/galaxy/datatypes/util/ +lib/galaxy/dataset_collections/ +lib/galaxy/datatypes/ +lib/galaxy/dependencies/ lib/galaxy/eggs/ -lib/galaxy/exceptions/__init__.py +lib/galaxy/exceptions/ lib/galaxy/external_services/__init__.py lib/galaxy/external_services/parameters.py lib/galaxy/external_services/result_handlers/basic.py lib/galaxy/external_services/result_handlers/__init__.py -lib/galaxy_ext/ lib/galaxy/forms/__init__.py lib/galaxy/jobs/actions/ lib/galaxy/jobs/command_factory.py @@ -113,15 +56,7 @@ lib/galaxy/managers/lddas.py lib/galaxy/managers/libraries.py lib/galaxy/managers/secured.py lib/galaxy/managers/taggable.py -lib/galaxy/model/__init__.py -lib/galaxy/model/item_attrs.py -lib/galaxy/model/mapping.py -lib/galaxy/model/metadata.py -lib/galaxy/model/migrate/ -lib/galaxy/model/orm/ -lib/galaxy/model/tool_shed_install/migrate/__init__.py -lib/galaxy/model/tool_shed_install/migrate/versions/ -lib/galaxy/model/util.py +lib/galaxy/model/ lib/galaxy/objectstore/pulsar.py lib/galaxy/objectstore/s3_multipart_upload.py lib/galaxy/openid/__init__.py @@ -129,7 +64,7 @@ lib/galaxy/quota/ lib/galaxy/sample_tracking/data_transfer.py lib/galaxy/sample_tracking/__init__.py lib/galaxy/sample_tracking/sample.py -lib/galaxy/security/validate_user_input.py +lib/galaxy/security/ lib/galaxy/tags/ lib/galaxy/tools/ lib/galaxy/util/ @@ -174,6 +109,7 @@ lib/galaxy/webapps/galaxy/api/samples.py lib/galaxy/webapps/galaxy/api/tools.py lib/galaxy/webapps/galaxy/api/tours.py lib/galaxy/webapps/galaxy/api/workflows.py +lib/galaxy/webapps/galaxy/config_watchers.py lib/galaxy/webapps/galaxy/controllers/async.py lib/galaxy/webapps/galaxy/controllers/data_manager.py lib/galaxy/webapps/galaxy/controllers/error.py @@ -186,7 +122,6 @@ lib/galaxy/webapps/galaxy/controllers/requests.py lib/galaxy/webapps/galaxy/controllers/search.py lib/galaxy/webapps/galaxy/controllers/tool_runner.py lib/galaxy/webapps/galaxy/controllers/userskeys.py -lib/galaxy/webapps/galaxy/config_watchers.py lib/galaxy/webapps/galaxy/__init__.py lib/galaxy/webapps/__init__.py lib/galaxy/webapps/reports/config.py @@ -209,13 +144,11 @@ lib/galaxy/webapps/tool_shed/util/__init__.py lib/galaxy/webapps/tool_shed/util/ratings_util.py lib/galaxy/webapps/tool_shed/util/shed_statistics.py lib/galaxy/webapps/util.py -lib/galaxy/workflow/extract.py -lib/galaxy/workflow/__init__.py -lib/galaxy/workflow/render.py -lib/galaxy/workflow/run.py -lib/galaxy/workflow/schedulers/ -lib/galaxy/workflow/steps.py -lib/galaxy/work/__init__.py +lib/galaxy/webhooks/ +lib/galaxy/work/ +lib/galaxy/workflow/ +lib/galaxy_ext/ +lib/log_tempfile.py lib/mimeparse.py lib/psyco_full.py lib/tool_shed/ @@ -366,6 +299,7 @@ test/unit/shed_unit/test_fabric_util.py test/unit/shed_unit/test_td_common_util.py test/unit/test_galaxy_transactions.py test/unit/test_lazy_process.py +test/unit/test_objectstore.py test/unit/test_routes.py test/unit/test_security_helper.py test/unit/test_sockets.py @@ -393,6 +327,5 @@ test/unit/web/base/__init__.py test/unit/web/framework/__init__.py test/unit/web/__init__.py test/unit/workflows/ -test/unit/test_objectstore.py tool_list.py tools/ diff --git a/.ci/py3_sources.txt b/.ci/py3_sources.txt index c82d1d3e09e..676a2225f6c 100644 --- a/.ci/py3_sources.txt +++ b/.ci/py3_sources.txt @@ -8,23 +8,7 @@ lib/galaxy/auth/ lib/galaxy/config.py lib/galaxy/containers/ lib/galaxy/dataset_collections/ -lib/galaxy/datatypes/assembly.py -lib/galaxy/datatypes/binary.py -lib/galaxy/datatypes/constructive_solid_geometry.py -lib/galaxy/datatypes/converters/ -lib/galaxy/datatypes/dataproviders/ -lib/galaxy/datatypes/data.py -lib/galaxy/datatypes/genetics.py -lib/galaxy/datatypes/images.py -lib/galaxy/datatypes/interval.py -lib/galaxy/datatypes/msa.py -lib/galaxy/datatypes/ngsindex.py -lib/galaxy/datatypes/proteomics.py -lib/galaxy/datatypes/registry.py -lib/galaxy/datatypes/sequence.py -lib/galaxy/datatypes/sniff.py -lib/galaxy/datatypes/tabular.py -lib/galaxy/datatypes/tracks.py +lib/galaxy/datatypes/ lib/galaxy/dependencies/ lib/galaxy/eggs/ lib/galaxy/exceptions/ @@ -32,12 +16,7 @@ lib/galaxy/external_services/ lib/galaxy/forms/ lib/galaxy/jobs/ lib/galaxy/managers/ -lib/galaxy/model/__init__.py -lib/galaxy/model/item_attrs.py -lib/galaxy/model/mapping.py -lib/galaxy/model/metadata.py -lib/galaxy/model/migrate/ -lib/galaxy/model/orm/now.py +lib/galaxy/model/ lib/galaxy/objectstore/ lib/galaxy/openid/ lib/galaxy/quota/ @@ -69,11 +48,9 @@ lib/galaxy/webapps/tool_shed/controllers/user.py lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py lib/galaxy/webapps/tool_shed/__init__.py lib/galaxy/webapps/tool_shed/util/ratings_util.py +lib/galaxy/webhooks/ lib/galaxy/work/ -lib/galaxy/workflow/extract.py -lib/galaxy/workflow/run.py -lib/galaxy/workflow/schedulers/ -lib/galaxy/workflow/steps.py +lib/galaxy/workflow/ lib/galaxy_ext/ lib/log_tempfile.py lib/psyco_full.py diff --git a/lib/galaxy/dataset_collections/matching.py b/lib/galaxy/dataset_collections/matching.py index 3398aec4a65..7230916a5d4 100644 --- a/lib/galaxy/dataset_collections/matching.py +++ b/lib/galaxy/dataset_collections/matching.py @@ -1,6 +1,10 @@ -from galaxy.util import bunch from galaxy import exceptions -from .structure import get_structure, leaf +from galaxy.util import bunch + +from .structure import ( + get_structure, + leaf +) CANNOT_MATCH_ERROR_MESSAGE = "Cannot match collection types." @@ -23,8 +27,8 @@ class CollectionsToMatch( object ): def has_collections( self ): return len( self.collections ) > 0 - def iteritems( self ): - return self.collections.iteritems() + def items( self ): + return self.collections.items() class MatchingCollections( object ): @@ -71,7 +75,7 @@ class MatchingCollections( object ): return None matching_collections = MatchingCollections() - for input_key, to_match in collections_to_match.iteritems(): + for input_key, to_match in collections_to_match.items(): hdca = to_match.hdca collection_type_description = collection_type_descriptions.for_collection_type( hdca.collection.collection_type ) subcollection_type = to_match.subcollection_type diff --git a/lib/galaxy/dataset_collections/registry.py b/lib/galaxy/dataset_collections/registry.py index d688d3af4df..16319fa38e2 100644 --- a/lib/galaxy/dataset_collections/registry.py +++ b/lib/galaxy/dataset_collections/registry.py @@ -1,7 +1,10 @@ -from .types import list -from .types import paired from galaxy import model +from .types import ( + list, + paired +) + PLUGIN_CLASSES = [list.ListDatasetCollectionType, paired.PairedDatasetCollectionType] diff --git a/lib/galaxy/dataset_collections/structure.py b/lib/galaxy/dataset_collections/structure.py index d2e7cc3ed25..889dc3c8ecb 100644 --- a/lib/galaxy/dataset_collections/structure.py +++ b/lib/galaxy/dataset_collections/structure.py @@ -124,7 +124,7 @@ class Tree( object ): def dict_map( func, input_dict ): - return dict( [ ( k, func(v) ) for k, v in input_dict.items() ] ) + return dict( ( k, func(v) ) for k, v in input_dict.items() ) def get_structure( dataset_collection_instance, collection_type_description, leaf_subcollection_type=None ): diff --git a/lib/galaxy/dataset_collections/types/list.py b/lib/galaxy/dataset_collections/types/list.py index 37dbee29a14..44c0576e29a 100644 --- a/lib/galaxy/dataset_collections/types/list.py +++ b/lib/galaxy/dataset_collections/types/list.py @@ -1,7 +1,7 @@ -from ..types import BaseDatasetCollectionType - from galaxy.model import DatasetCollectionElement +from ..types import BaseDatasetCollectionType + class ListDatasetCollectionType( BaseDatasetCollectionType ): """ A flat list of named elements. @@ -12,7 +12,7 @@ class ListDatasetCollectionType( BaseDatasetCollectionType ): pass def generate_elements( self, elements ): - for identifier, element in elements.iteritems(): + for identifier, element in elements.items(): association = DatasetCollectionElement( element=element, element_identifier=identifier, diff --git a/lib/galaxy/dataset_collections/types/paired.py b/lib/galaxy/dataset_collections/types/paired.py index 378e3ec9706..80cdbd5ef0d 100644 --- a/lib/galaxy/dataset_collections/types/paired.py +++ b/lib/galaxy/dataset_collections/types/paired.py @@ -1,7 +1,7 @@ -from ..types import BaseDatasetCollectionType - from galaxy.model import DatasetCollectionElement, HistoryDatasetAssociation +from ..types import BaseDatasetCollectionType + FORWARD_IDENTIFIER = "forward" REVERSE_IDENTIFIER = "reverse" diff --git a/lib/galaxy/datatypes/blast.py b/lib/galaxy/datatypes/blast.py index 2d68cb32eac..107202610f7 100644 --- a/lib/galaxy/datatypes/blast.py +++ b/lib/galaxy/datatypes/blast.py @@ -30,14 +30,16 @@ Covers the ``blastxml`` format and the BLAST databases. """ - import logging import os from time import sleep -from galaxy.datatypes.data import get_file_peek -from galaxy.datatypes.data import Data, Text -from galaxy.datatypes.xml import GenericXml +from .data import ( + Data, + get_file_peek, + Text +) +from .xml import GenericXml log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/chrominfo.py b/lib/galaxy/datatypes/chrominfo.py index 73ec75a40fc..179bf4a807e 100644 --- a/lib/galaxy/datatypes/chrominfo.py +++ b/lib/galaxy/datatypes/chrominfo.py @@ -1,7 +1,7 @@ from __future__ import absolute_import -import galaxy.datatypes.tabular import galaxy.datatypes.metadata +import galaxy.datatypes.tabular from galaxy.datatypes.metadata import MetadataElement diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py index 3fedd8572e8..07021532737 100644 --- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py @@ -5,18 +5,19 @@ from __future__ import division import sys from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict +from six import Iterator BLOCK_SIZE = 100 -class BedGraphReader: +class BedGraphReader(Iterator): def __init__( self, f ): self.f = f def __iter__( self ): return self - def next( self ): + def __next__( self ): while True: line = self.f.readline() if not line: @@ -46,7 +47,7 @@ def main(): # Fill array from reader d = array_tree_dict_from_reader( reader, {}, block_size=BLOCK_SIZE ) - for array_tree in d.itervalues(): + for array_tree in d.values(): array_tree.root.build_summary() FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) diff --git a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py index 9914b7d91b6..f77467683cd 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py +++ b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py @@ -10,9 +10,8 @@ Input: fasta Output: tabular """ - -import sys import os +import sys seq_hash = {} diff --git a/lib/galaxy/datatypes/converters/interval_to_fli.py b/lib/galaxy/datatypes/converters/interval_to_fli.py index 467fb90228a..831efddc695 100644 --- a/lib/galaxy/datatypes/converters/interval_to_fli.py +++ b/lib/galaxy/datatypes/converters/interval_to_fli.py @@ -89,7 +89,7 @@ def main(): out = open( out_fname, 'w' ) max_len = 0 entries = [] - for name in sorted( name_loc_dict.iterkeys() ): + for name in sorted( name_loc_dict.keys() ): loc = name_loc_dict[ name ] entry = '%s\t%s\t%s' % ( name.lower(), name, '%s:%i-%i' % ( loc[ 'contig' ], loc[ 'start' ], loc[ 'end' ] ) ) if len( entry ) > max_len: diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 049edc15fab..f571befcd84 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -3,9 +3,9 @@ from __future__ import print_function import os +import subprocess import sys import tempfile -import subprocess import time prog = "pbed_ldreduced_converter.py" diff --git a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py index 55266c6e7b3..8a361012d07 100644 --- a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py +++ b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py @@ -4,9 +4,8 @@ convert a ref.taxonomy file to a seq.taxonomy file Usage: %python ref_to_seq_taxonomy_converter.py """ - -import sys import re +import sys assert sys.version_info[:2] >= (2, 4) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index 6fe14fed055..804da49b90a 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -5,12 +5,12 @@ A wrapper script for converting SAM to BAM, with sorting. %prog input_filename.sam output_filename.bam """ -import os -import sys import optparse -import tempfile -import subprocess +import os import shutil +import subprocess +import sys +import tempfile from distutils.version import LooseVersion CHUNK_SIZE = 2 ** 20 # 1mb diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py index e8d76b428fb..922c24f7ede 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - """ Convert from VCF file to interval index file. """ @@ -7,9 +6,8 @@ from __future__ import division import optparse -from bx.interval_index_file import Indexes - import galaxy_utils.sequence.vcf +from bx.interval_index_file import Indexes def main(): diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py index 820ba7a1efa..ca9056258c0 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py @@ -20,7 +20,7 @@ def main(): # Fill array from reader d = array_tree_dict_from_reader( reader, {}, block_size=BLOCK_SIZE ) - for array_tree in d.itervalues(): + for array_tree in d.values(): array_tree.root.build_summary() FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py index b615b97c27c..90c887cea12 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py @@ -11,7 +11,10 @@ import sys import bx.wiggle -from galaxy.util.ucsc import UCSCOutWrapper, UCSCLimitException +from galaxy.util.ucsc import ( + UCSCLimitException, + UCSCOutWrapper +) def stop_err( msg ): diff --git a/lib/galaxy/datatypes/coverage.py b/lib/galaxy/datatypes/coverage.py index 5a3c7d60cbc..af70f8d8968 100644 --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -22,29 +22,6 @@ class LastzCoverage( Tabular ): MetadataElement( name="reverseCol", desc="Optional reverse read column", param=metadata.ColumnParameter, optional=True, no_value=0 ) MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - def get_track_window(self, dataset, data, start, end): - """ - Assumes we have a numpy file. - """ - # Maybe if we import here people will still be able to use Galaxy when numpy kills it - import numpy - - range = end - start - # Determine appropriate resolution to plot ~1000 points - resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) ) - # Restrict to valid range - resolution = min( resolution, 10000 ) - resolution = max( resolution, 1 ) - # Memory map the array (don't load all the data) - data = numpy.load( data ) - # Grab just what we need - t_start = math.floor( start / resolution ) - t_end = math.ceil( end / resolution ) - x = numpy.arange( t_start, t_end ) * resolution - y = data[ t_start : t_end ] - - return zip(x.tolist(), y.tolist()) - def get_track_resolution( self, dataset, start, end): range = end - start # Determine appropriate resolution to plot ~1000 points diff --git a/lib/galaxy/datatypes/dataproviders/chunk.py b/lib/galaxy/datatypes/dataproviders/chunk.py index 435c70d31f0..dc7a1b62739 100644 --- a/lib/galaxy/datatypes/dataproviders/chunk.py +++ b/lib/galaxy/datatypes/dataproviders/chunk.py @@ -4,16 +4,18 @@ Chunk (N number of bytes at M offset to a source's beginning) provider. Primarily for file sources but usable by any iterator that has both seek and read( N ). """ -import os import base64 -import base -import exceptions - import logging +import os + +from . import ( + base, + exceptions +) + log = logging.getLogger( __name__ ) -# ----------------------------------------------------------------------------- class ChunkDataProvider( base.DataProvider ): """ Data provider that yields chunks of data from its file. diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index bc97fc6f699..70d3a794662 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -2,9 +2,10 @@ Providers that provide lists of lists generally where each line of a source is further subdivided into multiple data (e.g. columns from a line). """ +import logging +import re from six.moves.urllib.parse import unquote_plus -import re from . import line @@ -15,7 +16,6 @@ TransposedColumnarDataProvider: provides each column as a single array - see existing visualizations/dataprovider/basic.ColumnDataProvider """ -import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/datatypes/dataproviders/dataset.py b/lib/galaxy/datatypes/dataproviders/dataset.py index 2ed8314fa48..3076e0218b0 100644 --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -5,18 +5,23 @@ Dataproviders that use either: - or provide data in some way relevant to bioinformatic data (e.g. parsing genomic regions from their source) """ - - -import base -import line -import column -import external -from galaxy.util import sqlite +import logging import sys -from bx import seq as bx_seq -from bx import wiggle as bx_wig -from bx import bbi as bx_bbi +from bx import ( + bbi as bx_bbi, + seq as bx_seq, + wiggle as bx_wig +) + +from galaxy.util import sqlite + +from . import ( + base, + column, + external, + line +) _TODO = """ use bx as much as possible @@ -25,7 +30,6 @@ gff3 hierarchies change SamtoolsDataProvider to use pysam """ -import logging log = logging.getLogger( __name__ ) @@ -150,7 +154,7 @@ class DatasetDataProvider( base.DataProvider ): """ region_column_names = ( 'chromCol', 'startCol', 'endCol' ) region_indices = [ self.get_metadata_column_index_by_name( name ) for name in region_column_names ] - if check and not all( map( lambda i: i is not None, region_indices) ): + if check and not all( _ is not None for _ in region_indices ): raise ValueError( "Could not determine proper column indices for chrom, start, end: %s" % ( str( region_indices ) ) ) return region_indices @@ -296,7 +300,7 @@ class GenomicRegionDataProvider( column.ColumnarDataProvider ): if end_column is None: end_column = dataset_source.get_metadata_column_index_by_name( 'endCol' ) indeces = [ chrom_column, start_column, end_column ] - if not all( map( lambda i: i is not None, indeces ) ): + if not all( _ is not None for _ in indeces ): raise ValueError( "Could not determine proper column indeces for" + " chrom, start, end: %s" % ( str( indeces ) ) ) kwargs.update({ 'indeces' : indeces }) diff --git a/lib/galaxy/datatypes/dataproviders/decorators.py b/lib/galaxy/datatypes/dataproviders/decorators.py index 99e01e1934a..93cfec13396 100644 --- a/lib/galaxy/datatypes/dataproviders/decorators.py +++ b/lib/galaxy/datatypes/dataproviders/decorators.py @@ -14,11 +14,12 @@ DataProvider related decorators. # adapted from: http://stackoverflow.com # /questions/14095616/python-can-i-programmatically-decorate-class-methods-from-a-class-instance -from functools import wraps -import urllib2 import copy - import logging +from functools import wraps + +from six.moves.urllib.parse import unquote + log = logging.getLogger( __name__ ) _DATAPROVIDER_CLASS_MAP_KEY = 'dataproviders' @@ -65,7 +66,7 @@ def has_dataproviders( cls ): # scan for methods with dataprovider names and add them to the map # note: this has a 'cascading' effect # where it's possible to override a super's provider with a sub's - for attr_key, attr_value in cls.__dict__.iteritems(): + for attr_key, attr_value in cls.__dict__.items(): # can't use isinstance( attr_value, MethodType ) bc of wrapping if( ( callable( attr_value ) ) and ( not attr_key.startswith( "__" ) ) and @@ -129,7 +130,7 @@ def _parse_query_string_settings( query_kwargs, settings=None ): 'float' : float, 'bool' : bool, 'list:str' : lambda s: list_from_query_string( s ), - 'list:escaped' : lambda s: [ urllib2.unquote( e ) for e in list_from_query_string( s ) ], + 'list:escaped' : lambda s: [ unquote( e ) for e in list_from_query_string( s ) ], 'list:int' : lambda s: [ int( i ) for i in list_from_query_string( s ) ], } settings = settings or {} diff --git a/lib/galaxy/datatypes/dataproviders/external.py b/lib/galaxy/datatypes/dataproviders/external.py index 4d5eac41072..81751e5c963 100644 --- a/lib/galaxy/datatypes/dataproviders/external.py +++ b/lib/galaxy/datatypes/dataproviders/external.py @@ -2,14 +2,18 @@ Data providers that iterate over a source that is not in memory or not in a file. """ - -import base import gzip -import line +import logging import subprocess import tempfile -import urllib -import urllib2 + +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import urlopen + +from . import ( + base, + line +) _TODO = """ YAGNI: ftp, image, cryptos, sockets @@ -17,7 +21,6 @@ job queue admin: admin server log rgx/stats, ps aux """ -import logging log = logging.getLogger( __name__ ) @@ -105,13 +108,13 @@ class URLDataProvider( base.DataProvider ): self.method = method self.data = data or {} - encoded_data = urllib.urlencode( self.data ) + encoded_data = urlencode( self.data ) if method == 'GET': self.url += '?%s' % ( encoded_data ) - opened = urllib2.urlopen( url ) + opened = urlopen( url ) elif method == 'POST': - opened = urllib2.urlopen( url, encoded_data ) + opened = urlopen( url, encoded_data ) else: raise ValueError( 'Not a valid method: %s' % ( method ) ) diff --git a/lib/galaxy/datatypes/dataproviders/hierarchy.py b/lib/galaxy/datatypes/dataproviders/hierarchy.py index 818f3953cef..6626f5c19a2 100644 --- a/lib/galaxy/datatypes/dataproviders/hierarchy.py +++ b/lib/galaxy/datatypes/dataproviders/hierarchy.py @@ -1,14 +1,17 @@ """ Dataproviders that iterate over lines from their sources. """ +import logging +from xml.etree.ElementTree import ( + Element, + iterparse +) -import line -from xml.etree.ElementTree import Element, iterparse +from . import line _TODO = """ """ -import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/datatypes/dataproviders/line.py b/lib/galaxy/datatypes/dataproviders/line.py index 0809773450e..652bd408d67 100644 --- a/lib/galaxy/datatypes/dataproviders/line.py +++ b/lib/galaxy/datatypes/dataproviders/line.py @@ -1,12 +1,12 @@ """ Dataproviders that iterate over lines from their sources. """ - import collections +import logging import re + from . import base -import logging log = logging.getLogger( __name__ ) _TODO = """ diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index a8e75e9e76e..0dfc3dd014f 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -1,16 +1,24 @@ # Contains objects for using external display applications import logging -import urllib -from six import string_types -from urllib import quote_plus from copy import deepcopy -from galaxy.util import parse_xml, string_as_bool +from six import string_types +from six.moves.urllib.parse import quote_plus + +from galaxy.util import ( + parse_xml, + string_as_bool +) from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for -from parameters import DisplayApplicationParameter, DisplayApplicationDataParameter, DEFAULT_DATASET_NAME -from util import encode_dataset_user + +from .parameters import ( + DEFAULT_DATASET_NAME, + DisplayApplicationDataParameter, + DisplayApplicationParameter +) +from .util import encode_dataset_user log = logging.getLogger( __name__ ) @@ -50,8 +58,8 @@ class DisplayApplicationLink( object ): action="display_application", dataset_id=dataset_hash, user_id=user_hash, - app_name=urllib.quote_plus( self.display_application.id ), - link_name=urllib.quote_plus( self.id ), + app_name=quote_plus( self.display_application.id ), + link_name=quote_plus( self.id ), app_action=None ) def get_inital_values( self, data, trans ): @@ -60,7 +68,7 @@ class DisplayApplicationLink( object ): else: rval = odict() rval.update( { 'BASE_URL': trans.request.base, 'APP': trans.app } ) # trans automatically appears as a response, need to add properties of trans that we want here - for key, value in BASE_PARAMS.iteritems(): # add helper functions/variables + for key, value in BASE_PARAMS.items(): # add helper functions/variables rval[ key ] = value rval[ DEFAULT_DATASET_NAME ] = data # always have the display dataset name available return rval @@ -70,7 +78,7 @@ class DisplayApplicationLink( object ): other_values[ 'DATASET_HASH' ] = dataset_hash other_values[ 'USER_HASH' ] = user_hash ready = True - for name, param in self.parameters.iteritems(): + for name, param in self.parameters.items(): assert name not in other_values, "The display parameter '%s' has been defined more than once." % name if param.ready( other_values ): if name in app_kwds and param.allow_override: @@ -142,7 +150,7 @@ class DynamicDisplayApplicationBuilder( object ): max_col = max( id_col, name_col ) dynamic_params = {} if data_table is not None: - max_col = max( [ max_col ] + data_table.columns.values() ) + max_col = max( [ max_col ] + list(data_table.columns.values()) ) for key, value in data_table.columns.items(): dynamic_params[key] = { 'column': value, 'split': False, 'separator': ',' } for dynamic_param in elem.findall( 'dynamic_param' ): @@ -174,7 +182,7 @@ class DynamicDisplayApplicationBuilder( object ): new_elem.set( 'id', fields[id_col] ) new_elem.set( 'name', fields[name_col] ) dynamic_values = {} - for key, attributes in dynamic_params.iteritems(): + for key, attributes in dynamic_params.items(): value = fields[ attributes[ 'column' ] ] if attributes['split']: value = value.split( attributes['separator'] ) @@ -210,7 +218,7 @@ class PopulatedDisplayApplicationLink( object ): def preparing_display( self ): if not self.ready: - return self.link.parameters[ self.parameters.keys()[ -1 ] ].is_preparing( self.parameters ) + return self.link.parameters[ list(self.parameters.keys())[-1] ].is_preparing( self.parameters ) return False def prepare_display( self ): @@ -218,8 +226,8 @@ class PopulatedDisplayApplicationLink( object ): found_last = False if not self.ready and not self.preparing_display(): other_values = self.parameters - for name, param in self.link.parameters.iteritems(): - if found_last or other_values.keys()[ -1 ] == name: # found last parameter to be populated + for name, param in self.link.parameters.items(): + if found_last or list(other_values.keys())[-1] == name: # found last parameter to be populated found_last = True value = param.prepare( other_values, self.dataset_hash, self.user_hash, self.trans ) rval.append( { 'name': name, 'value': value, 'param': param } ) @@ -231,7 +239,7 @@ class PopulatedDisplayApplicationLink( object ): def get_prepare_steps( self, datasets_only=True ): rval = [] - for name, param in self.link.parameters.iteritems(): + for name, param in self.link.parameters.items(): if datasets_only and not isinstance( param, DisplayApplicationDataParameter ): continue value = self.parameters.get( name, None ) @@ -243,7 +251,7 @@ class PopulatedDisplayApplicationLink( object ): return fill_template( self.link.url.text, context=self.parameters ) def get_param_name_by_url( self, url ): - for name, parameter in self.link.parameters.iteritems(): + for name, parameter in self.link.parameters.items(): if parameter.build_url( self.parameters ) == url: return name raise ValueError( "Unknown URL parameter name provided: %s" % url ) @@ -301,7 +309,7 @@ class DisplayApplication( object ): def filter_by_dataset( self, data, trans ): self._check_and_reload() filtered = DisplayApplication( self.id, self.name, self.app, version=self.version ) - for link_name, link_value in self.links.iteritems(): + for link_name, link_value in self.links.items(): if link_value.filter_by_dataset( data, trans ): filtered.links[link_name] = link_value return filtered @@ -318,13 +326,11 @@ class DisplayApplication( object ): # We will not allow changing the id at this time (we'll need to fix several mappings upstream to handle this case) assert attr_dict.get( 'id' ) == self.id, ValueError( "You cannot reload a Display application where the ID has changed. You will need to restart the server instead." ) # clear old links - for key in self.links.keys(): - del self.links[key] + self.links = {} # clear data table versions: - for key in self._data_table_versions.keys(): - del self._data_table_versions[ key ] + self._data_table_versions = {} # Set new attributes - for key, value in attr_dict.iteritems(): + for key, value in attr_dict.items(): setattr( self, key, value ) # Load new links self._load_links_from_elem( elem ) @@ -334,7 +340,7 @@ class DisplayApplication( object ): self._data_table_versions[ table_name ] = version def _requires_reload( self ): - for key, value in self._data_table_versions.iteritems(): + for key, value in self._data_table_versions.items(): table = self.app.tool_data_tables.get( key, None ) if table and not table.is_current_version( value ): return True diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index b771aed0278..97e0b4d27d3 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -1,10 +1,12 @@ # Contains parameters that are used in Display Applications -import urllib +import mimetypes + +from six.moves.urllib.parse import quote_plus + from galaxy.util import string_as_bool from galaxy.util.bunch import Bunch from galaxy.util.template import fill_template from galaxy.web import url_for -import mimetypes DEFAULT_DATASET_NAME = 'dataset' @@ -110,7 +112,7 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): if target_ext and not converted_dataset: if isinstance( data, DisplayDataValueWrapper ): data = data.value - new_data = data.datatype.convert_dataset( trans, data, target_ext, return_output=True, visible=False ).values()[0] + new_data = next(iter(data.datatype.convert_dataset( trans, data, target_ext, return_output=True, visible=False ).values())) new_data.hid = data.hid new_data.name = data.name trans.sa_session.add( new_data ) @@ -193,8 +195,8 @@ class DisplayParameterValueWrapper( object ): action="display_application", dataset_id=self._dataset_hash, user_id=self._user_hash, - app_name=urllib.quote_plus( self.parameter.link.display_application.id ), - link_name=urllib.quote_plus( self.parameter.link.id ), + app_name=quote_plus( self.parameter.link.display_application.id ), + link_name=quote_plus( self.parameter.link.id ), app_action=self.action_name, action_param=self._url ) ) diff --git a/lib/galaxy/datatypes/graph.py b/lib/galaxy/datatypes/graph.py index 4b4f8083eca..913cec78a0a 100644 --- a/lib/galaxy/datatypes/graph.py +++ b/lib/galaxy/datatypes/graph.py @@ -1,15 +1,17 @@ """ Graph content classes. """ +import logging -import data -import tabular -import xml - -import dataproviders from galaxy.util import simplegraph -import logging +from . import ( + data, + dataproviders, + tabular, + xml +) + log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 1992479e8bb..c774fd1e8a1 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -7,7 +7,6 @@ import os import sys import tempfile -import numpy from bx.intervals.io import GenomicIntervalReader, ParseError from six.moves.urllib.parse import quote_plus @@ -335,20 +334,6 @@ class Interval( Tabular ): except: return False - def get_track_window(self, dataset, data, start, end): - """ - Assumes the incoming track data is sorted already. - """ - window = list() - for record in data: - fields = record.rstrip("\n\r").split("\t") - record_chrom = fields[dataset.metadata.chromCol - 1] - record_start = int(fields[dataset.metadata.startCol - 1]) - record_end = int(fields[dataset.metadata.endCol - 1]) - if record_start < end and record_end > start: - window.append( (record_chrom, record_start, record_end) ) # Yes I did want to use a generator here, but it doesn't work downstream - return window - def get_track_resolution( self, dataset, start, end): return None @@ -1259,26 +1244,6 @@ class Wiggle( Tabular, _RemoteCallMixin ): except: return False - def get_track_window(self, dataset, data, start, end): - """ - Assumes we have a numpy file. - """ - range = end - start - # Determine appropriate resolution to plot ~1000 points - resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) ) - # Restrict to valid range - resolution = min( resolution, 100000 ) - resolution = max( resolution, 1 ) - # Memory map the array (don't load all the data) - data = numpy.load( data ) - # Grab just what we need - t_start = math.floor( start / resolution ) - t_end = math.ceil( end / resolution ) - x = numpy.arange( t_start, t_end ) * resolution - y = data[ t_start : t_end ] - - return list(zip(x.tolist(), y.tolist())) - def get_track_resolution( self, dataset, start, end): range = end - start # Determine appropriate resolution to plot ~1000 points diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index 8482ad1d7c8..703e143e6c8 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -1,17 +1,18 @@ # -*- coding: utf-8 -*- - -from galaxy.datatypes import data import logging -from galaxy.datatypes.sniff import get_headers -from galaxy.datatypes.data import get_file_peek -from galaxy.datatypes.tabular import Tabular -from galaxy.datatypes.binary import Binary -from galaxy.datatypes.xml import GenericXml -import subprocess import os +import subprocess +from galaxy.datatypes import ( + data, + metadata +) +from galaxy.datatypes.binary import Binary +from galaxy.datatypes.data import get_file_peek from galaxy.datatypes.metadata import MetadataElement -from galaxy.datatypes import metadata +from galaxy.datatypes.sniff import get_headers +from galaxy.datatypes.tabular import Tabular +from galaxy.datatypes.xml import GenericXml log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/mothur.py b/lib/galaxy/datatypes/mothur.py index 4df70361da1..c6a6ce44cb1 100644 --- a/lib/galaxy/datatypes/mothur.py +++ b/lib/galaxy/datatypes/mothur.py @@ -2,11 +2,12 @@ Mothur Metagenomics Datatypes """ import logging -import sys import re -from galaxy.datatypes.sniff import get_headers -from galaxy.datatypes.metadata import MetadataElement +import sys + from galaxy.datatypes.data import Text +from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes.sniff import get_headers from galaxy.datatypes.tabular import Tabular log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index f3b97b9e85a..3c7d0644ecc 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -2,9 +2,9 @@ Neo4j Composite Dataset """ import logging -import sys -import shutil import os +import shutil +import sys from galaxy.datatypes.data import Data from galaxy.datatypes.images import Html @@ -29,7 +29,7 @@ class Neo4j(Html): 'Files for Composite Dataset (%s)

\ This composite dataset is composed of the following files:

    ' % ( self.file_ext)] - for composite_name, composite_file in self.get_composite_files(dataset=dataset).iteritems(): + for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): opt_text = '' if composite_file.optional: opt_text = ' (optional)' diff --git a/lib/galaxy/datatypes/qualityscore.py b/lib/galaxy/datatypes/qualityscore.py index 9723d7340c2..9fda147c4de 100644 --- a/lib/galaxy/datatypes/qualityscore.py +++ b/lib/galaxy/datatypes/qualityscore.py @@ -1,9 +1,10 @@ """ Qualityscore class """ - -import data import logging + +from . import data + log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index b2a82e9f8bf..d86d53c8fdb 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -3,8 +3,9 @@ File format detector """ from __future__ import absolute_import -import gzip import bz2 +import codecs +import gzip import logging import os import re @@ -13,13 +14,13 @@ import sys import tempfile import zipfile -from encodings import search_function as encodings_search_function from six import text_type from galaxy import util -from galaxy.util import multi_byte +from galaxy.datatypes.binary import Binary from galaxy.util import ( compression_utils, + multi_byte, unicodify ) from galaxy.util.checkers import ( @@ -28,7 +29,6 @@ from galaxy.util.checkers import ( is_bz2, is_gzip ) -from galaxy.datatypes.binary import Binary log = logging.getLogger(__name__) @@ -48,7 +48,9 @@ def stream_to_open_named_file( stream, fd, filename, source_encoding=None, sourc is_compressed = False is_binary = False is_multi_byte = False - if not target_encoding or not encodings_search_function( target_encoding ): + try: + codecs.lookup(target_encoding) + except: target_encoding = util.DEFAULT_ENCODING # utf-8 if not source_encoding: source_encoding = util.DEFAULT_ENCODING # sys.getdefaultencoding() would mimic old behavior (defaults to ascii) diff --git a/lib/galaxy/datatypes/triples.py b/lib/galaxy/datatypes/triples.py index af82f3a6465..1b8df11dd5d 100644 --- a/lib/galaxy/datatypes/triples.py +++ b/lib/galaxy/datatypes/triples.py @@ -1,12 +1,15 @@ """ Triple format classes """ -import re -import data import logging -import xml -import text -import binary +import re + +from . import ( + binary, + data, + text, + xml +) log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 502f405cc29..ec96a28e381 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -6,6 +6,8 @@ import copy from bx.intervals.io import GenomicInterval, GenomicIntervalReader, MissingFieldError, NiceReaderWrapper, ParseError from bx.tabular.io import Comment, Header +from six import Iterator + from galaxy.util.odict import odict @@ -117,7 +119,7 @@ class GFFIntervalToBEDReaderWrapper( NiceReaderWrapper ): return interval -class GFFReaderWrapper( NiceReaderWrapper ): +class GFFReaderWrapper( Iterator, NiceReaderWrapper ): # Iterator can be removed after bx-python library is ported to Python3 """ Reader wrapper for GFF files. @@ -149,15 +151,14 @@ class GFFReaderWrapper( NiceReaderWrapper ): self.default_strand, fix_strand=self.fix_strand ) return interval - # For Python3 this needs to be changed to __next__() after bx-python library is ported too - def next( self ): + def __next__( self ): """ Returns next GFFFeature. """ # # Helper function. # - def handle_parse_error( parse_error ): + def handle_parse_error( e ): """ Actions to take when ParseError found. """ if self.outstream: if self.print_delegate and hasattr(self.print_delegate, "__call__"): @@ -434,19 +435,16 @@ def read_unordered_gtf( iterator, strict=False ): chroms_features = {} for count, intervals in enumerate( feature_intervals.values() ): # Sort intervals by start position. - intervals.sort( lambda a, b: cmp( a.start, b.start ) ) + intervals.sort(key=lambda _: _.start) feature = GFFFeature( None, intervals=intervals ) if feature.chrom not in chroms_features: chroms_features[ feature.chrom ] = [] chroms_features[ feature.chrom ].append( feature ) # Sort features by chrom, start position. - chroms_features_sorted = [] - for chrom_features in chroms_features.values(): - chroms_features_sorted.append( chrom_features ) - chroms_features_sorted.sort( lambda a, b: cmp( a[0].chrom, b[0].chrom ) ) + chroms_features_sorted = sorted(chroms_features.values(), key=lambda _: _[0].chrom) for features in chroms_features_sorted: - features.sort( lambda a, b: cmp( a.start, b.start ) ) + features.sort(key=lambda _: _.start) # Yield comments first, then features. # FIXME: comments can appear anywhere in file, not just the beginning. diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index ce26b0ce0c9..4a1c22b949d 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -1,10 +1,13 @@ """ XML format classes """ -import re -import data import logging -import dataproviders +import re + +from . import ( + data, + dataproviders +) log = logging.getLogger(__name__) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 883b750dcb9..677e5b210ae 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -1,11 +1,11 @@ """ Determine what optional dependencies are needed. """ -import pkg_resources - from os.path import dirname, join from xml.etree import ElementTree +import pkg_resources + from galaxy.util import asbool from galaxy.util.properties import load_app_properties diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py index 60574a98e02..63661ba84e9 100644 --- a/lib/galaxy/model/base.py +++ b/lib/galaxy/model/base.py @@ -2,10 +2,17 @@ Shared model and mapping code between Galaxy and Tool Shed, trying to generalize to generic database connections. """ +from inspect import ( + getmembers, + isclass +) + +from sqlalchemy.orm import ( + scoped_session, + sessionmaker +) -from sqlalchemy.orm import scoped_session, sessionmaker from galaxy.util.bunch import Bunch -from inspect import getmembers, isclass # TODO: Refactor this to be a proper class, not a bunch. diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 21bf881003f..54b793637bc 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -3,17 +3,21 @@ import copy import json import logging import uuid - -from sys import getsizeof -from itertools import chain from collections import deque +from itertools import chain +from sys import getsizeof import sqlalchemy +from sqlalchemy.ext.mutable import Mutable +from sqlalchemy.types import ( + CHAR, + LargeBinary, + String, + TypeDecorator +) from galaxy import app from galaxy.util.aliaspickler import AliasPickleModule -from sqlalchemy.types import CHAR, LargeBinary, String, TypeDecorator -from sqlalchemy.ext.mutable import Mutable log = logging.getLogger( __name__ ) @@ -234,6 +238,7 @@ def total_size(o, handlers={}, verbose=False): """ def dict_handler(d): return chain.from_iterable(d.items()) + all_handlers = { tuple: iter, list: iter, deque: iter, @@ -268,7 +273,7 @@ class MetadataType( JSONType ): def process_bind_param(self, value, dialect): if value is not None: if app.app and app.app.config.max_metadata_value_size: - for k, v in value.items(): + for k, v in list(value.items()): sz = total_size(v) if sz > app.app.config.max_metadata_value_size: del value[k] diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index 6f4372ab104..7ddde634a9d 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -32,12 +32,28 @@ import parsley from sqlalchemy import and_ from sqlalchemy.orm import aliased -from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, - History, Library, LibraryFolder, LibraryDataset, StoredWorkflowTagAssociation, - StoredWorkflow, HistoryTagAssociation, HistoryDatasetAssociationTagAssociation, - ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, - JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, - Page, PageRevision) +from galaxy.model import ( + ExtendedMetadata, + ExtendedMetadataIndex, + History, + HistoryAnnotationAssociation, + HistoryDatasetAssociation, + HistoryDatasetAssociationTagAssociation, + HistoryTagAssociation, + Job, + JobParameter, + JobToInputDatasetAssociation, + JobToInputLibraryDatasetAssociation, + JobToOutputDatasetAssociation, + Library, + LibraryDataset, + LibraryDatasetDatasetAssociation, + LibraryFolder, + Page, + PageRevision, + StoredWorkflow, + StoredWorkflowTagAssociation +) from galaxy.model.tool_shed_install import ToolVersion log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index ba0cc4b6c79..bd13fbb6c3b 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -1,10 +1,12 @@ import logging import os -from galaxy.util.dictifiable import Dictifiable -from galaxy.util.bunch import Bunch + +from six.moves.urllib.parse import urljoin + from galaxy.util import asbool +from galaxy.util.bunch import Bunch +from galaxy.util.dictifiable import Dictifiable from tool_shed.util import common_util -from urlparse import urljoin log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/model/tool_shed_install/mapping.py b/lib/galaxy/model/tool_shed_install/mapping.py index bbff370c07f..8c4e5f438f9 100644 --- a/lib/galaxy/model/tool_shed_install/mapping.py +++ b/lib/galaxy/model/tool_shed_install/mapping.py @@ -1,11 +1,27 @@ +from sqlalchemy import ( + Boolean, + Column, + DateTime, + ForeignKey, + Integer, + MetaData, + String, + Table, + TEXT +) +from sqlalchemy.orm import ( + mapper, + relation +) + from galaxy.model import tool_shed_install as install_model -from sqlalchemy import MetaData -from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Table, TEXT -from sqlalchemy.orm import relation, mapper -from galaxy.model.custom_types import JSONType, TrimmedString -from galaxy.model.orm.now import now from galaxy.model.base import ModelMapping +from galaxy.model.custom_types import ( + JSONType, + TrimmedString +) from galaxy.model.orm.engine_factory import build_engine +from galaxy.model.orm.now import now metadata = MetaData() diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index c7d16395886..478f8fa6136 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -1,14 +1,17 @@ -import sys -import os.path import logging +import os.path +import sys -# from sqlalchemy import * -from sqlalchemy import create_engine -from sqlalchemy import MetaData -from sqlalchemy import Table - +from migrate.versioning import ( + repository, + schema +) +from sqlalchemy import ( + create_engine, + MetaData, + Table +) from sqlalchemy.exc import NoSuchTableError -from migrate.versioning import repository, schema log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index 9db9fe5cd34..2283e6f85f8 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -6,7 +6,7 @@ import logging import socket from datetime import datetime, timedelta -from sqlalchemy import and_, false, or_, not_ +from sqlalchemy import and_, false, not_, or_ from sqlalchemy.orm import eagerload_all from galaxy.util import listify @@ -1008,7 +1008,7 @@ class GalaxyRBACAgent( RBACAgent ): # Add the new permissions on library_item for item_class, permission_class in self.library_item_assocs: if isinstance( library_item, item_class ): - for action, roles in permissions.items(): + for action, roles in list(permissions.items()): if isinstance( action, Action ): action = action.action for role_assoc in [ permission_class( action, library_item, role ) for role in roles ]: diff --git a/lib/galaxy/security/passwords.py b/lib/galaxy/security/passwords.py index 79dc573e120..14e81314af8 100644 --- a/lib/galaxy/security/passwords.py +++ b/lib/galaxy/security/passwords.py @@ -1,10 +1,11 @@ -import hmac import hashlib -from struct import Struct -from operator import xor -from itertools import starmap -from os import urandom +import hmac from base64 import b64encode +from itertools import starmap +from operator import xor +from os import urandom +from struct import Struct + from galaxy.util import safe_str_cmp SALT_LENGTH = 12 @@ -77,6 +78,6 @@ def pbkdf2_bin( data, salt, iterations=1000, keylen=24, hashfunc=None ): rv = u = _pseudorandom(salt + _pack_int(block)) for i in range(iterations - 1): u = _pseudorandom(''.join(map(chr, u))) - rv = starmap( xor, zip( rv, u ) ) # Python < 2.6.8: starmap requires function inputs to be tuples, so we need to use zip instead of izip + rv = starmap( xor, zip( rv, u ) ) buf.extend(rv) return ''.join(map(chr, buf))[:keylen] diff --git a/lib/galaxy/webhooks/__init__.py b/lib/galaxy/webhooks/__init__.py index 28012e4af50..8d2c8b25e79 100644 --- a/lib/galaxy/webhooks/__init__.py +++ b/lib/galaxy/webhooks/__init__.py @@ -1,10 +1,10 @@ """ This module manages loading of Galaxy webhooks. """ - -import os -import yaml import logging +import os + +import yaml from galaxy.util import config_directories_from_setting diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index 4252ef9f4a0..8822a1b4ccf 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -1,7 +1,7 @@ from galaxy.managers.context import ( ProvidesAppContext, - ProvidesUserContext, - ProvidesHistoryContext + ProvidesHistoryContext, + ProvidesUserContext ) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index fd6571660f7..33432e1094f 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -3,34 +3,44 @@ Modules used in building workflows """ import logging from json import loads -from xml.etree.ElementTree import Element -from xml.etree.ElementTree import XML +from xml.etree.ElementTree import ( + Element, + XML +) -from galaxy import exceptions, model, web -from galaxy.exceptions import ToolMissingException +from galaxy import ( + exceptions, + model, + web +) from galaxy.dataset_collections import matching +from galaxy.exceptions import ToolMissingException from galaxy.jobs.actions.post import ActionBox from galaxy.model import PostJobAction +from galaxy.tools import ( + DefaultToolState, + ToolInputsNotReadyException +) from galaxy.tools.execute import execute -from galaxy.tools.parameters import check_param -from galaxy.tools.parameters import params_to_incoming -from galaxy.tools.parameters import visit_input_values +from galaxy.tools.parameters import ( + check_param, + params_to_incoming, + visit_input_values +) from galaxy.tools.parameters.basic import ( - parameter_types, BooleanToolParameter, - TextToolParameter, - SelectToolParameter, DataCollectionToolParameter, DataToolParameter, + parameter_types, RuntimeValue, + SelectToolParameter, + TextToolParameter, workflow_building_modes ) from galaxy.tools.parameters.wrapped import make_dict_copy -from galaxy.tools import DefaultToolState -from galaxy.tools import ToolInputsNotReadyException -from galaxy.util.odict import odict from galaxy.util.bunch import Bunch from galaxy.util.json import safe_loads +from galaxy.util.odict import odict from tool_shed.util import common_util log = logging.getLogger( __name__ ) @@ -346,7 +356,7 @@ class InputModule( WorkflowModule ): # Web controller may set copy_inputs_to_history, API controller always sets # inputs. if invocation.copy_inputs_to_history: - for input_dataset_hda in step_outputs.values(): + for input_dataset_hda in list(step_outputs.values()): content_type = input_dataset_hda.history_content_type if content_type == "dataset": new_hda = input_dataset_hda.copy( copy_children=True ) @@ -362,7 +372,7 @@ class InputModule( WorkflowModule ): # so do that now so dependent steps can be recalculated. In the future # everything should come in from the API and this can be eliminated. if not invocation.has_input_for_step( step.id ): - content = step_outputs.values()[ 0 ] + content = next(iter(step_outputs.values())) if content: invocation.add_input( content, step.id ) progress.set_outputs_for_input( step, step_outputs ) @@ -610,7 +620,7 @@ class ToolModule( WorkflowModule ): super( ToolModule, self ).save_to_step( step ) step.tool_id = self.tool_id step.tool_version = self.get_version() - for k, v in self.post_job_actions.iteritems(): + for k, v in self.post_job_actions.items(): pja = self.__to_pja( k, v, step ) self.trans.sa_session.add( pja ) @@ -666,7 +676,7 @@ class ToolModule( WorkflowModule ): def get_data_outputs( self ): data_outputs = [] if self.tool: - for name, tool_output in self.tool.outputs.iteritems(): + for name, tool_output in self.tool.outputs.items(): extra_kwds = {} if tool_output.collection: extra_kwds["collection"] = True @@ -907,7 +917,7 @@ class ToolModule( WorkflowModule ): # job actions for this execution. flush_required = False effective_post_job_actions = step.post_job_actions[:] - for key, value in self.runtime_post_job_actions.iteritems(): + for key, value in self.runtime_post_job_actions.items(): effective_post_job_actions.append( self.__to_pja( key, value, None ) ) for pja in effective_post_job_actions: if pja.action_type in ActionBox.immediate_actions: diff --git a/lib/galaxy/workflow/render.py b/lib/galaxy/workflow/render.py index 1534d94d936..c9b4bd4412a 100644 --- a/lib/galaxy/workflow/render.py +++ b/lib/galaxy/workflow/render.py @@ -96,8 +96,8 @@ class WorkflowCanvas( object ): else: # Take any key / value pair available in out_conn_index_dict. # A problem will result if the dictionary is empty. - if out_conn_index_dict.keys(): - key = out_conn_index_dict.keys()[0] + if out_conn_index_dict: + key = next(iter(out_conn_index_dict.keys())) out_conn_pos = self.out_pos[ out_pos_index ][ key ] adjusted = ( out_conn_pos[ 0 ] + self.widths[ output_dict[ 'id' ] ], out_conn_pos[ 1 ] ) self.text.append( svgwrite.shapes.Circle(center=(out_conn_pos[ 0 ] + self.widths[ output_dict[ 'id' ] ] - MARGIN, @@ -130,7 +130,7 @@ class WorkflowCanvas( object ): fill = "#EBD9B2" width = self.widths[ step_dict[ 'id' ] ] self.add_boxes( step_dict, width, fill ) - for conn, output_dict in step_dict[ 'input_connections' ].iteritems(): + for conn, output_dict in step_dict[ 'input_connections' ].items(): self.add_connection( step_dict, conn, output_dict ) def populate_data_for_step( self, step, module_name, module_data_inputs, module_data_outputs, tool_errors=None ): diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py index 56fe32fdc3c..8eac6e37b8c 100644 --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -1,14 +1,15 @@ +import logging import uuid -from galaxy import exceptions -from galaxy import model - +from galaxy import ( + exceptions, + model +) from galaxy.managers import histories from galaxy.tools.parameters.meta import expand_workflow_inputs INPUT_STEP_TYPES = [ 'data_input', 'data_collection_input', 'parameter_input' ] -import logging log = logging.getLogger( __name__ ) @@ -147,8 +148,7 @@ def _flatten_step_params( param_dict, prefix="" ): # a complex value object versus something that maps to child parameters # better than the hack or searching for src and id here. new_params = {} - keys = param_dict.keys()[:] - for key in keys: + for key in list(param_dict.keys()): if prefix: effective_key = "%s|%s" % ( prefix, key ) else: @@ -244,7 +244,7 @@ def build_workflow_run_configs( trans, workflow, payload ): steps_by_id = workflow.steps_by_id # Set workflow inputs. - for key, input_dict in normalized_inputs.iteritems(): + for key, input_dict in normalized_inputs.items(): step = steps_by_id[key] if step.type == 'parameter_input': continue @@ -353,13 +353,13 @@ def workflow_run_config_to_request( trans, run_config, workflow ): ) replacement_dict = run_config.replacement_dict - for name, value in replacement_dict.iteritems(): + for name, value in replacement_dict.items(): add_parameter( name=name, value=value, type=param_types.REPLACEMENT_PARAMETERS, ) - for step_id, content in run_config.inputs.iteritems(): + for step_id, content in run_config.inputs.items(): workflow_invocation.add_input( content, step_id ) add_parameter( "copy_inputs_to_history", "true" if run_config.copy_inputs_to_history else "false", param_types.META_PARAMETERS ) diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 5197a3637da..0d5f0e2d307 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -1,16 +1,14 @@ -import os -import time import logging +import os import threading - +import time from xml.etree import ElementTree +import galaxy.workflow.schedulers from galaxy import model from galaxy.util import plugin_config from galaxy.util.handlers import ConfiguresHandlers -import galaxy.workflow.schedulers - log = logging.getLogger( __name__ ) DEFAULT_SCHEDULER_ID = "default" # well actually this should be called DEFAULT_DEFAULT_SCHEDULER_ID... @@ -77,7 +75,7 @@ class WorkflowSchedulingManager( object, ConfiguresHandlers ): return self.__has_handlers.get_handler( None, index=random_index ) def shutdown( self ): - for workflow_scheduler in self.workflow_schedulers.itervalues(): + for workflow_scheduler in self.workflow_schedulers.values(): try: workflow_scheduler.shutdown() except Exception: @@ -103,7 +101,7 @@ class WorkflowSchedulingManager( object, ConfiguresHandlers ): return workflow_invocation def __start_schedulers( self ): - for workflow_scheduler in self.workflow_schedulers.itervalues(): + for workflow_scheduler in self.workflow_schedulers.values(): workflow_scheduler.startup( self.app ) def __plugins_dict( self ): @@ -185,7 +183,7 @@ class WorkflowRequestMonitor( object ): def __monitor( self ): to_monitor = self.workflow_scheduling_manager.active_workflow_schedulers while self.active: - for workflow_scheduler_id, workflow_scheduler in to_monitor.iteritems(): + for workflow_scheduler_id, workflow_scheduler in to_monitor.items(): if not self.active: return diff --git a/lib/log_tempfile.py b/lib/log_tempfile.py index 1c50aeb3ee1..ccc0ea52d77 100644 --- a/lib/log_tempfile.py +++ b/lib/log_tempfile.py @@ -1,9 +1,8 @@ # override tempfile methods for debugging - +import logging import tempfile import traceback -import logging log = logging.getLogger( __name__ )