diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index b9c3491444a..0f435eb5fab 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -14,6 +14,7 @@ lib/galaxy/dataset_collections/__init__.py lib/galaxy/dataset_collections/structure.py lib/galaxy/dataset_collections/subcollections.py lib/galaxy/dataset_collections/type_description.py +lib/galaxy/dataset_collections/types/__init__.py lib/galaxy/datatypes/assembly.py lib/galaxy/datatypes/binary.py lib/galaxy/datatypes/checkers.py @@ -93,16 +94,7 @@ lib/galaxy/jobs/runners/pulsar.py lib/galaxy/jobs/runners/slurm.py lib/galaxy/jobs/runners/state_handlers/ lib/galaxy/jobs/runners/tasks.py -lib/galaxy/jobs/runners/util/cli/factory.py -lib/galaxy/jobs/runners/util/cli/job/__init__.py -lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py -lib/galaxy/jobs/runners/util/cli/job/torque.py -lib/galaxy/jobs/runners/util/cli/shell/__init__.py -lib/galaxy/jobs/runners/util/drmaa/ -lib/galaxy/jobs/runners/util/env.py -lib/galaxy/jobs/runners/util/external.py -lib/galaxy/jobs/runners/util/__init__.py -lib/galaxy/jobs/runners/util/job_script/ +lib/galaxy/jobs/runners/util/ lib/galaxy/jobs/splitters/basic.py lib/galaxy/jobs/splitters/__init__.py lib/galaxy/jobs/stock_rules.py @@ -266,16 +258,7 @@ lib/galaxy/security/validate_user_input.py lib/galaxy/tags/ lib/galaxy/tools/ lib/galaxy/util/ -lib/galaxy_utils/__init__.py -lib/galaxy/util/sleeper.py -lib/galaxy/util/specs.py -lib/galaxy_utils/sequence/fasta.py -lib/galaxy_utils/sequence/fastq.py -lib/galaxy_utils/sequence/__init__.py -lib/galaxy_utils/sequence/transform.py -lib/galaxy_utils/sequence/vcf.py -lib/galaxy/util/template.py -lib/galaxy/util/ucsc.py +lib/galaxy_utils/ lib/galaxy/version.py lib/galaxy/visualization/data_providers/basic.py lib/galaxy/visualization/data_providers/cigar.py diff --git a/.ci/py3_sources.txt b/.ci/py3_sources.txt index 25e55544477..69539cf362d 100644 --- a/.ci/py3_sources.txt +++ b/.ci/py3_sources.txt @@ -65,7 +65,7 @@ lib/galaxy/webapps/tool_shed/util/ratings_util.py lib/galaxy/work/ lib/galaxy/workflow/extract.py lib/galaxy/workflow/run.py -lib/galaxy/workflow/schedulers/core.py +lib/galaxy/workflow/schedulers/ lib/galaxy/workflow/steps.py lib/galaxy_ext/ lib/galaxy_utils/ diff --git a/lib/galaxy/auth/providers/__init__.py b/lib/galaxy/auth/providers/__init__.py index 054bb45fd49..514aa1b3eb8 100644 --- a/lib/galaxy/auth/providers/__init__.py +++ b/lib/galaxy/auth/providers/__init__.py @@ -3,13 +3,14 @@ Created on 15/07/2014 @author: Andrew Robinson """ - import abc +import six + +@six.add_metaclass(abc.ABCMeta) class AuthProvider(object): """A base class for all Auth Providers.""" - __metaclass__ = abc.ABCMeta @abc.abstractproperty def plugin_type(self): diff --git a/lib/galaxy/dataset_collections/types/__init__.py b/lib/galaxy/dataset_collections/types/__init__.py index 239b070ae09..cb0a870b958 100644 --- a/lib/galaxy/dataset_collections/types/__init__.py +++ b/lib/galaxy/dataset_collections/types/__init__.py @@ -1,13 +1,18 @@ -from galaxy import exceptions -from abc import ABCMeta -from abc import abstractmethod - import logging +from abc import ( + ABCMeta, + abstractmethod +) + +import six + +from galaxy import exceptions + log = logging.getLogger( __name__ ) +@six.add_metaclass(ABCMeta) class DatasetCollectionType(object): - __metaclass__ = ABCMeta @abstractmethod def generate_elements( self, dataset_instances ): diff --git a/lib/galaxy/jobs/datasets.py b/lib/galaxy/jobs/datasets.py index 9fcea9c1292..36643482bb7 100644 --- a/lib/galaxy/jobs/datasets.py +++ b/lib/galaxy/jobs/datasets.py @@ -2,13 +2,17 @@ Utility classes allowing Job interface to reason about datasets. """ import os.path -from abc import ABCMeta -from abc import abstractmethod +from abc import ( + ABCMeta, + abstractmethod +) + +import six def dataset_path_rewrites( dataset_paths ): - dataset_paths_with_rewrites = filter( lambda path: getattr( path, "false_path", None ), dataset_paths ) - return dict( [ ( dp.real_path, dp ) for dp in dataset_paths_with_rewrites ] ) + dataset_paths_with_rewrites = [path for path in dataset_paths if getattr(path, "false_path", None)] + return dict( ( dp.real_path, dp ) for dp in dataset_paths_with_rewrites ) class DatasetPath( object ): @@ -49,9 +53,9 @@ class DatasetPath( object ): return dataset_path +@six.add_metaclass(ABCMeta) class DatasetPathRewriter( object ): """ Used by runner to rewrite paths. """ - __metaclass__ = ABCMeta @abstractmethod def rewrite_dataset_path( self, dataset, dataset_type ): diff --git a/lib/galaxy/jobs/metrics/instrumenters/__init__.py b/lib/galaxy/jobs/metrics/instrumenters/__init__.py index b996daf5c79..6e772a1ae63 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/__init__.py +++ b/lib/galaxy/jobs/metrics/instrumenters/__init__.py @@ -3,19 +3,21 @@ These are responsible for collecting and formatting a coherent set of metrics. """ import os.path +from abc import ( + ABCMeta, + abstractmethod +) -from abc import ABCMeta -from abc import abstractmethod +import six from ...metrics import formatting - INSTRUMENT_FILE_PREFIX = "__instrument" +@six.add_metaclass(ABCMeta) class InstrumentPlugin( object ): """Describes how to instrument job scripts and retrieve collected metrics.""" - __metaclass__ = ABCMeta formatter = formatting.JobMetricFormatter() @property diff --git a/lib/galaxy/jobs/runners/util/cli/__init__.py b/lib/galaxy/jobs/runners/util/cli/__init__.py index cdd2143e1f2..b16ca63b67f 100644 --- a/lib/galaxy/jobs/runners/util/cli/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/__init__.py @@ -1,8 +1,11 @@ """ """ from glob import glob -from os.path import basename, join from os import getcwd +from os.path import ( + basename, + join +) DEFAULT_SHELL_PLUGIN = 'LocalShell' @@ -61,7 +64,7 @@ class CliInterface(object): raise ValueError(ERROR_MESSAGE_NO_JOB_PLUGIN) job_plugin_class = self.cli_job_interfaces.get(job_plugin, None) if not job_plugin_class: - raise ValueError(ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN % (job_plugin, self.cli_job_interfaces.keys())) + raise ValueError(ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN % (job_plugin, list(self.cli_job_interfaces.keys()))) job_interface = job_plugin_class(**job_params) return job_interface diff --git a/lib/galaxy/jobs/runners/util/cli/job/__init__.py b/lib/galaxy/jobs/runners/util/cli/job/__init__.py index 683506694b7..c6af843a2aa 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/job/__init__.py @@ -1,11 +1,16 @@ """ Abstract base class for cli job plugins. """ -from abc import ABCMeta, abstractmethod +from abc import ( + ABCMeta, + abstractmethod +) + +import six +@six.add_metaclass(ABCMeta) class BaseJobExec(object): - __metaclass__ = ABCMeta @abstractmethod def __init__(self, **params): diff --git a/lib/galaxy/jobs/runners/util/cli/job/slurm.py b/lib/galaxy/jobs/runners/util/cli/job/slurm.py index 5ae3cb5f780..a6a88ead1f3 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/slurm.py +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py @@ -1,10 +1,9 @@ # A simple CLI runner for slurm that can be used when running Galaxy from a # non-submit host and using a Slurm cluster. +from logging import getLogger from ..job import BaseJobExec -from logging import getLogger - try: from galaxy.model import Job job_states = Job.states diff --git a/lib/galaxy/jobs/runners/util/cli/shell/__init__.py b/lib/galaxy/jobs/runners/util/cli/shell/__init__.py index e14f4a8dda9..3d55f07aba6 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/__init__.py @@ -1,11 +1,16 @@ """ Abstract base class for runners which execute commands via a shell. """ -from abc import ABCMeta, abstractmethod +from abc import ( + ABCMeta, + abstractmethod +) + +import six +@six.add_metaclass(ABCMeta) class BaseShellExec(object): - __metaclass__ = ABCMeta @abstractmethod def __init__(self, *args, **kwargs): diff --git a/lib/galaxy/jobs/runners/util/cli/shell/local.py b/lib/galaxy/jobs/runners/util/cli/shell/local.py index 9fc5904e80b..4e610470bdf 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/local.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/local.py @@ -1,11 +1,17 @@ +from logging import getLogger +from subprocess import ( + PIPE, + Popen +) from tempfile import TemporaryFile from time import sleep -from subprocess import Popen, PIPE from ..shell import BaseShellExec -from ....util import Bunch, kill_pid +from ....util import ( + Bunch, + kill_pid +) -from logging import getLogger log = getLogger(__name__) TIMEOUT_ERROR_MESSAGE = u'Execution timed out' diff --git a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py index b6fdcc8c678..ecfffd0f284 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py @@ -1,6 +1,7 @@ +from logging import getLogger + from .local import LocalShell -from logging import getLogger log = getLogger(__name__) __all__ = ('RemoteShell', 'SecureShell', 'GlobusSecureShell') diff --git a/lib/galaxy/jobs/runners/util/condor/__init__.py b/lib/galaxy/jobs/runners/util/condor/__init__.py index d0103caee34..a8a9003f01e 100644 --- a/lib/galaxy/jobs/runners/util/condor/__init__.py +++ b/lib/galaxy/jobs/runners/util/condor/__init__.py @@ -1,7 +1,14 @@ """ Condor helper utilities. """ -from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError +from subprocess import ( + CalledProcessError, + check_call, + PIPE, + Popen, + STDOUT +) + from ..external import parse_external_id DEFAULT_QUERY_CLASSAD = dict( diff --git a/lib/galaxy/jobs/runners/util/kill.py b/lib/galaxy/jobs/runners/util/kill.py index 52022466552..c0dbd913b13 100644 --- a/lib/galaxy/jobs/runners/util/kill.py +++ b/lib/galaxy/jobs/runners/util/kill.py @@ -1,10 +1,10 @@ import os from platform import system -from time import sleep from subprocess import Popen +from time import sleep try: - from psutil import Process, NoSuchProcess + from psutil import NoSuchProcess, Process except ImportError: """ Don't make psutil a strict requirement, but use if available. """ Process = None diff --git a/lib/galaxy/jobs/runners/util/retry.py b/lib/galaxy/jobs/runners/util/retry.py index 4d9da3c2cef..fa4dbbec39d 100644 --- a/lib/galaxy/jobs/runners/util/retry.py +++ b/lib/galaxy/jobs/runners/util/retry.py @@ -1,7 +1,7 @@ +import logging from itertools import count from time import sleep -import logging log = logging.getLogger(__name__) DEFAULT_MAX_RETRIES = -1 # By default don't retry. diff --git a/lib/galaxy/jobs/runners/util/sudo.py b/lib/galaxy/jobs/runners/util/sudo.py index bb9e03dbd97..0cfca1619c2 100644 --- a/lib/galaxy/jobs/runners/util/sudo.py +++ b/lib/galaxy/jobs/runners/util/sudo.py @@ -1,6 +1,8 @@ import logging - -from subprocess import Popen, PIPE +from subprocess import ( + PIPE, + Popen +) SUDO_PATH = '/usr/bin/sudo' SUDO_PRESERVE_ENVIRONMENT_ARG = '-E' diff --git a/lib/galaxy/tools/deps/container_resolvers/__init__.py b/lib/galaxy/tools/deps/container_resolvers/__init__.py index c478bcce3a2..8e5e35d05a6 100644 --- a/lib/galaxy/tools/deps/container_resolvers/__init__.py +++ b/lib/galaxy/tools/deps/container_resolvers/__init__.py @@ -5,17 +5,18 @@ from abc import ( abstractproperty, ) +import six + from galaxy.util.dictifiable import Dictifiable +@six.add_metaclass(ABCMeta) class ContainerResolver(Dictifiable, object): """Description of a technique for resolving container images for tool execution.""" # Keys for dictification. dict_collection_visible_keys = ['resolver_type'] - __metaclass__ = ABCMeta - def __init__(self, app_info=None, **kwds): """Default initializer for ``ContainerResolver`` subclasses.""" self.app_info = app_info diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index d30349645cd..dba15fe5306 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -279,8 +279,8 @@ class JobInfo(object): self.job_directory_type = job_directory_type # "galaxy" or "pulsar" +@six.add_metaclass(ABCMeta) class Container( object ): - __metaclass__ = ABCMeta def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description): self.container_id = container_id @@ -445,8 +445,9 @@ class NullContainer(object): def __init__(self): pass - def __nonzero__(self): + def __bool__(self): return False + __nonzero__ = __bool__ NULL_CONTAINER = NullContainer() diff --git a/lib/galaxy/tools/deps/installable.py b/lib/galaxy/tools/deps/installable.py index 68489dc351c..0d2d955d490 100644 --- a/lib/galaxy/tools/deps/installable.py +++ b/lib/galaxy/tools/deps/installable.py @@ -2,13 +2,14 @@ import logging import os - from abc import ( ABCMeta, abstractmethod, abstractproperty, ) +import six + from galaxy.util.filelock import ( FileLock, FileLockException @@ -17,11 +18,10 @@ from galaxy.util.filelock import ( log = logging.getLogger(__name__) +@six.add_metaclass(ABCMeta) class InstallableContext(object): """Represent a directory/configuration of something that can be installed.""" - __metaclass__ = ABCMeta - @abstractmethod def is_installed(self): """Return bool indicating if the configured software is installed.""" diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index eb6fecbeaed..044af469363 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -5,6 +5,7 @@ from abc import ( abstractproperty, ) +import six import yaml from galaxy.util import listify @@ -13,6 +14,7 @@ from galaxy.util.dictifiable import Dictifiable from ..requirements import ToolRequirement +@six.add_metaclass(ABCMeta) class DependencyResolver(Dictifiable, object): """Abstract description of a technique for resolving container images for tool execution.""" @@ -26,7 +28,6 @@ class DependencyResolver(Dictifiable, object): disabled = False resolves_simple_dependencies = True config_options = {} - __metaclass__ = ABCMeta @abstractmethod def resolve( self, requirement, **kwds ): @@ -51,12 +52,12 @@ class MultipleDependencyResolver: """ +@six.add_metaclass(ABCMeta) class ListableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate the dependency resolver can iterate over its dependencies and generate requirements. """ - __metaclass__ = ABCMeta @abstractmethod def list_dependencies(self): @@ -87,7 +88,7 @@ class MappableDependencyResolver: def _mapping_file_to_list(mapping_file): with open(mapping_file, "r") as f: raw_mapping = yaml.load(f) or [] - return map(RequirementMapping.from_dict, raw_mapping) + return map(RequirementMapping.from_dict, raw_mapping) def _expand_mappings(self, requirement): for mapping in self._mappings: @@ -134,13 +135,13 @@ class RequirementMapping(object): return RequirementMapping(from_name, from_version, to_name, to_version) +@six.add_metaclass(ABCMeta) class SpecificationAwareDependencyResolver: """Mix this into a :class:`DependencyResolver` to implement URI specification matching. Allows adapting generic requirements to more specific URIs - to tailor name or version to specified resolution system. """ - __metaclass__ = ABCMeta @abstractmethod def _expand_specs(self, requirement): @@ -178,11 +179,11 @@ class SpecificationPatternDependencyResolver: return requirement +@six.add_metaclass(ABCMeta) class InstallableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate the dependency resolver can attempt to install new dependencies. """ - __metaclass__ = ABCMeta @abstractmethod def install_dependency(self, name, version, type, **kwds): @@ -191,9 +192,9 @@ class InstallableDependencyResolver: """ +@six.add_metaclass(ABCMeta) class Dependency(Dictifiable, object): dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version', 'cacheable'] - __metaclass__ = ABCMeta cacheable = False @abstractmethod diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py index 969012a8a90..b16f807ae7e 100644 --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -1,17 +1,19 @@ -from abc import ABCMeta -from abc import abstractmethod +from abc import ( + ABCMeta, + abstractmethod +) import six NOT_IMPLEMENTED_MESSAGE = "Galaxy tool format does not yet support this tool feature." +@six.add_metaclass(ABCMeta) @six.python_2_unicode_compatible class ToolSource(object): """ This interface represents an abstract source to parse tool information from. """ - __metaclass__ = ABCMeta default_is_multi_byte = False @abstractmethod @@ -202,8 +204,8 @@ class PagesSource(object): return True +@six.add_metaclass(ABCMeta) class PageSource(object): - __metaclass__ = ABCMeta def parse_display(self): return None @@ -213,8 +215,8 @@ class PageSource(object): """ Return a list of InputSource objects. """ +@six.add_metaclass(ABCMeta) class InputSource(object): - __metaclass__ = ABCMeta default_optional = False def elem(self): diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py index 6e7fe290185..8b82af33692 100644 --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -1,11 +1,15 @@ -from abc import ABCMeta -from abc import abstractmethod +from abc import ( + ABCMeta, + abstractmethod +) + +import six +@six.add_metaclass(ABCMeta) class ToolLineage(object): """ """ - __metaclass__ = ABCMeta @abstractmethod def get_versions( self, reverse=False ): diff --git a/lib/galaxy/tools/toolbox/tags.py b/lib/galaxy/tools/toolbox/tags.py index fab34a7f11f..33f1adf50a2 100644 --- a/lib/galaxy/tools/toolbox/tags.py +++ b/lib/galaxy/tools/toolbox/tags.py @@ -1,9 +1,12 @@ """ Handle details of tool tagging - perhaps a deprecated feature. """ import logging +from abc import ( + ABCMeta, + abstractmethod +) -from abc import ABCMeta -from abc import abstractmethod +import six log = logging.getLogger( __name__ ) @@ -18,8 +21,8 @@ def tool_tag_manager( app ): return NullToolTagManager() +@six.add_metaclass(ABCMeta) class AbstractToolTagManager( object ): - __metaclass__ = ABCMeta @abstractmethod def reset_tags( self ): diff --git a/lib/galaxy/util/object_wrapper.py b/lib/galaxy/util/object_wrapper.py index 3f7374f0078..d6593fb3a5c 100644 --- a/lib/galaxy/util/object_wrapper.py +++ b/lib/galaxy/util/object_wrapper.py @@ -6,8 +6,8 @@ import inspect import logging import string import sys - from numbers import Number + try: from types import NoneType except ImportError: @@ -56,6 +56,7 @@ from types import ( ModuleType, TracebackType, ) + from six.moves import ( copyreg as copy_reg, UserDict @@ -307,7 +308,7 @@ class SafeStringWrapper( object ): # Skip __slots__ - # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + # Don't need to define a metaclass, we'll use the helper function to handle with subclassing for e.g. isinstance() # Revisit: # __instancecheck__ diff --git a/lib/galaxy/workflow/schedulers/__init__.py b/lib/galaxy/workflow/schedulers/__init__.py index a7836e16198..ec5ff668745 100644 --- a/lib/galaxy/workflow/schedulers/__init__.py +++ b/lib/galaxy/workflow/schedulers/__init__.py @@ -2,10 +2,15 @@ for workflow scheduling is highly experimental and the interface required for scheduling plugins will almost certainly change. """ -from abc import ABCMeta -from abc import abstractmethod +from abc import ( + ABCMeta, + abstractmethod +) + +import six +@six.add_metaclass(ABCMeta) class WorkflowSchedulingPlugin( object ): """ A plugin defining how Galaxy should schedule plugins. By default plugins are passive and should monitor Galaxy's work queue for @@ -13,7 +18,6 @@ class WorkflowSchedulingPlugin( object ): the scheduling plugin should be forced (i.e. if scheduling happen all at once or the request will be stored and monitored outside of Galaxy.) """ - __metaclass__ = ABCMeta @property @abstractmethod @@ -30,8 +34,8 @@ class WorkflowSchedulingPlugin( object ): """ +@six.add_metaclass(ABCMeta) class ActiveWorkflowSchedulingPlugin( WorkflowSchedulingPlugin ): - __metaclass__ = ABCMeta @abstractmethod def schedule( self, workflow_invocation ): diff --git a/lib/galaxy_utils/sequence/sequence.py b/lib/galaxy_utils/sequence/sequence.py index 37a0a832347..cd3a109aaf0 100644 --- a/lib/galaxy_utils/sequence/sequence.py +++ b/lib/galaxy_utils/sequence/sequence.py @@ -1,8 +1,9 @@ # Dan Blankenberg -import transform import string from copy import deepcopy +from . import transform + class SequencingRead( object ): color_space_converter = transform.ColorSpaceConverter()