diff --git a/lib/galaxy/datatypes/anvio.py b/lib/galaxy/datatypes/anvio.py index 8428ccb4d97..6825be459e0 100644 --- a/lib/galaxy/datatypes/anvio.py +++ b/lib/galaxy/datatypes/anvio.py @@ -6,6 +6,7 @@ import glob import logging import os import sys +from typing import Optional from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.text import Html @@ -79,7 +80,7 @@ class AnvioComposite(Html): class AnvioDB(AnvioComposite): """Class for AnvioDB database files.""" - _anvio_basename = None + _anvio_basename: Optional[str] = None MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) file_ext = 'anvio_db' diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index eb7e43cc320..0a15f7dbbd1 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -14,6 +14,7 @@ import tempfile import zipfile from collections import OrderedDict from json import dumps +from typing import Optional import h5py import pysam @@ -51,7 +52,7 @@ class Binary(data.Data): def register_unsniffable_binary_ext(ext): """Deprecated method.""" - def set_peek(self, dataset, is_multi_byte=False): + def set_peek(self, dataset, **kwd): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = 'binary data' @@ -272,7 +273,7 @@ class BamNative(CompressedArchive): edam_format = "format_2572" edam_data = "data_0863" file_ext = "unsorted.bam" - sort_flag = None + sort_flag: Optional[str] = None MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None) MetadataElement(name="sort_order", default=None, desc="Sort Order", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None) @@ -1122,7 +1123,7 @@ class GmxBinary(Binary): Base class for GROMACS binary files - xtc, trr, cpt """ - magic_number = None # variables to be overwritten in the child class + magic_number: Optional[int] = None # variables to be overwritten in the child class file_ext = "" def sniff(self, filename): diff --git a/lib/galaxy/datatypes/blast.py b/lib/galaxy/datatypes/blast.py index 8037fe58362..fa29a8076fe 100644 --- a/lib/galaxy/datatypes/blast.py +++ b/lib/galaxy/datatypes/blast.py @@ -89,6 +89,7 @@ class BlastXml(GenericXml): return False return True + @staticmethod def merge(split_files, output_file): """Merging multiple XML files is non-trivial and must be done in subclasses.""" if len(split_files) == 1: @@ -167,7 +168,6 @@ class BlastXml(GenericXml): h.close() out.write(" \n") out.write("\n") - merge = staticmethod(merge) class _BlastDb(Data): diff --git a/lib/galaxy/datatypes/constructive_solid_geometry.py b/lib/galaxy/datatypes/constructive_solid_geometry.py index 21379c16914..35ab83304f7 100644 --- a/lib/galaxy/datatypes/constructive_solid_geometry.py +++ b/lib/galaxy/datatypes/constructive_solid_geometry.py @@ -1,6 +1,10 @@ +# TODO: revisit ignoring type and write some tests for this, the multi-inheritance in this +# this file is challenging, it should be broken into true mixins. +# type: ignore """ Constructive Solid Geometry file formats. """ + import abc from galaxy import util diff --git a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py index f77467683cd..424a9fa90fa 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py +++ b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py @@ -13,8 +13,6 @@ Output: tabular import os import sys -seq_hash = {} - def __main__(): infile = sys.argv[1] diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index e4bb22a9fad..f5410a558c2 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -7,6 +7,7 @@ import string import tempfile from collections import OrderedDict from inspect import isclass +from typing import Any, Dict, Optional import webob.exc from markupsafe import escape @@ -116,27 +117,27 @@ class Data(metaclass=DataMeta): # Add metadata elements MetadataElement(name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, no_value="?") # Stores the set of display applications, and viewing methods, supported by this datatype - supported_display_apps = {} + supported_display_apps: Dict[str, Any] = {} # If False, the peek is regenerated whenever a dataset of this type is copied copy_safe_peek = True # The dataset contains binary data --> do not space_to_tab or convert newlines, etc. # Allow binary file uploads of this type when True. is_binary = True # Composite datatypes - composite_type = None - composite_files = OrderedDict() + composite_type: Optional[str] = None + composite_files: Dict[str, Any] = OrderedDict() primary_file_name = 'index' # Allow user to change between this datatype and others. If left to None, # datatype change is allowed if the datatype is not composite. - allow_datatype_change = None + allow_datatype_change: Optional[bool] = None # A per datatype setting (inherited): max file size (in bytes) for setting optional metadata _max_optional_metadata_filesize = None # Trackster track type. - track_type = None + track_type: Optional[str] = None # Data sources. - data_sources = {} + data_sources: Dict[str, str] = {} def __init__(self, **kwd): """Initialize the datatype""" @@ -180,7 +181,7 @@ class Data(metaclass=DataMeta): if copy_from: dataset.metadata = copy_from.metadata - def set_meta(self, dataset, overwrite=True, **kwd): + def set_meta(self, dataset: Any, overwrite=True, **kwd): """Unimplemented method, allows guessing of metadata from contents of file""" return True @@ -681,11 +682,11 @@ class Data(metaclass=DataMeta): return key @property - def writable_files(self, dataset=None): + def writable_files(self): files = OrderedDict() if self.composite_type != 'auto_primary_file': files[self.primary_file_name] = self.__new_composite_file(self.primary_file_name) - for key, value in self.get_composite_files(dataset=dataset).items(): + for key, value in self.get_composite_files().items(): files[key] = value return files @@ -718,6 +719,7 @@ class Data(metaclass=DataMeta): datatype_classes = tuple(datatype if isclass(datatype) else datatype.__class__ for datatype in target_datatypes) return isinstance(self, datatype_classes) + @staticmethod def merge(split_files, output_file): """ Merge files with copy.copyfileobj() will not hit the @@ -732,8 +734,6 @@ class Data(metaclass=DataMeta): for fsrc in split_files: shutil.copyfileobj(open(fsrc, 'rb'), fdst) - merge = staticmethod(merge) - def get_visualizations(self, dataset): """ Returns a list of visualizations for datatype. @@ -844,7 +844,7 @@ class Text(Data): data_lines = None return data_lines - def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=True): + def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=True, **kwd): """ Set the peek. This method is used by various subclasses of Text. """ @@ -879,6 +879,7 @@ class Text(Data): dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ Split the input files by line. @@ -947,7 +948,6 @@ class Text(Data): f.close() if part_file: part_file.close() - split = classmethod(split) # ------------- Dataproviders @dataproviders.decorators.dataprovider_factory('line', dataproviders.line.FilteredLineDataProvider.settings) diff --git a/lib/galaxy/datatypes/dataproviders/base.py b/lib/galaxy/datatypes/dataproviders/base.py index 920affff1f2..22cd392d930 100644 --- a/lib/galaxy/datatypes/dataproviders/base.py +++ b/lib/galaxy/datatypes/dataproviders/base.py @@ -9,7 +9,7 @@ Base class(es) for all DataProviders. import logging from collections import deque - +from typing import Dict from . import exceptions @@ -74,7 +74,7 @@ class DataProvider(metaclass=HasSettings): # a definition of expected types for keyword arguments sent to __init__ # useful for controlling how query string dictionaries can be parsed into correct types for __init__ # empty in this base class - settings = {} + settings: Dict[str, str] = {} def __init__(self, source, **kwargs): """ diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index 0eca661e278..cdf22276a15 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -1,5 +1,6 @@ # Contains parameters that are used in Display Applications import mimetypes +from typing import Optional from urllib.parse import quote_plus from galaxy.util import string_as_bool @@ -12,7 +13,7 @@ DEFAULT_DATASET_NAME = 'dataset' class DisplayApplicationParameter: """ Abstract Class for Display Application Parameters """ - type = None + type: Optional[str] = None @classmethod def from_elem(cls, elem, link): diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index c3823e9c7bf..1eaa4fb81da 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -125,6 +125,7 @@ class SDF(GenericMolFile): """ dataset.metadata.number_of_molecules = count_special_lines(r"^\$\$\$\$$", dataset.file_name) + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ Split the input files by molecule records. @@ -172,7 +173,6 @@ class SDF(GenericMolFile): except Exception as e: log.error('Unable to split files: %s', unicodify(e)) raise - split = classmethod(split) @build_sniff_from_prefix @@ -208,6 +208,7 @@ class MOL2(GenericMolFile): """ dataset.metadata.number_of_molecules = count_special_lines("@MOLECULE", dataset.file_name) + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ Split the input files by molecule records. @@ -259,7 +260,6 @@ class MOL2(GenericMolFile): except Exception as e: log.error('Unable to split files: %s', unicodify(e)) raise - split = classmethod(split) @build_sniff_from_prefix @@ -293,6 +293,7 @@ class FPS(GenericMolFile): """ dataset.metadata.number_of_molecules = count_special_lines('^#', dataset.file_name, invert=True) + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ Split the input files by fingerprint records. @@ -338,8 +339,8 @@ class FPS(GenericMolFile): except Exception as e: log.error('Unable to split files: %s', unicodify(e)) raise - split = classmethod(split) + @staticmethod def merge(split_files, output_file): """ Merging fps files requires merging the header manually. @@ -363,7 +364,6 @@ class FPS(GenericMolFile): # line is no header and not a comment, we assume the first header is written to out and we set 'first' to False first = False out.write(line) - merge = staticmethod(merge) class OBFS(Binary): @@ -846,6 +846,7 @@ class CML(GenericXml): return True + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ Split the input files by molecule records. @@ -902,8 +903,8 @@ class CML(GenericXml): except Exception as e: log.error('Unable to split files: %s', unicodify(e)) raise - split = classmethod(split) + @staticmethod def merge(split_files, output_file): """ Merging CML files. @@ -938,4 +939,3 @@ class CML(GenericXml): if molecule_found: out.write(line) out.write("\n") - merge = staticmethod(merge) diff --git a/lib/galaxy/datatypes/msa.py b/lib/galaxy/datatypes/msa.py index 9b4eaa9bd68..cba811b94f9 100644 --- a/lib/galaxy/datatypes/msa.py +++ b/lib/galaxy/datatypes/msa.py @@ -169,6 +169,7 @@ class Stockholm_1_0(Text): """ dataset.metadata.number_of_models = generic_util.count_special_lines('^#[[:space:]+]STOCKHOLM[[:space:]+]1.0', dataset.file_name) + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ @@ -218,7 +219,6 @@ class Stockholm_1_0(Text): except Exception as e: log.error('Unable to split files: %s', unicodify(e)) raise - split = classmethod(split) @build_sniff_from_prefix diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index b3711698d2d..4c75f7f7101 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -120,6 +120,7 @@ class Sequence(data.Text): dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' + @staticmethod def get_sequences_per_file(total_sequences, split_params): if split_params['split_mode'] == 'number_of_parts': # legacy basic mode - split into a specified number of parts @@ -138,8 +139,8 @@ class Sequence(data.Text): else: raise Exception('Unsupported split mode %s' % split_params['split_mode']) return sequences_per_file - get_sequences_per_file = staticmethod(get_sequences_per_file) + @classmethod def do_slow_split(cls, input_datasets, subdir_generator_function, split_params): # count the sequences so we can split # TODO: if metadata is present, take the number of lines / 4 @@ -152,8 +153,8 @@ class Sequence(data.Text): sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params) return cls.write_split_files(input_datasets, None, subdir_generator_function, sequences_per_file) - do_slow_split = classmethod(do_slow_split) + @classmethod def do_fast_split(cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params): data = json.load(open(toc_file_datasets[0].file_name)) sections = data['sections'] @@ -162,8 +163,8 @@ class Sequence(data.Text): total_sequences += int(section['sequences']) sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params) return cls.write_split_files(input_datasets, toc_file_datasets, subdir_generator_function, sequences_per_file) - do_fast_split = classmethod(do_fast_split) + @classmethod def write_split_files(cls, input_datasets, toc_file_datasets, subdir_generator_function, sequences_per_file): directories = [] @@ -195,7 +196,6 @@ class Sequence(data.Text): json.dump(split_data, f) start_sequence += sequences_per_file[part_no] return directories - write_split_files = classmethod(write_split_files) def split(cls, input_datasets, subdir_generator_function, split_params): """Split a generic sequence file (not sensible or possible, see subclasses).""" @@ -203,6 +203,7 @@ class Sequence(data.Text): return None raise NotImplementedError("Can't split generic sequence files") + @staticmethod def get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count): """ Uses a Table of Contents dict, parsed from an FQTOC file, to come up with a set of @@ -270,8 +271,8 @@ class Sequence(data.Text): raise Exception('%s sequences not found in file' % sequence_count) return result - get_split_commands_with_toc = staticmethod(get_split_commands_with_toc) + @staticmethod def get_split_commands_sequential(is_compressed, input_name, output_name, start_sequence, sequence_count): """ Does a brain-dead sequential scan & extract of certain sequences @@ -290,7 +291,6 @@ class Sequence(data.Text): cmd += ' > "%s"' % output_name return [cmd] - get_split_commands_sequential = staticmethod(get_split_commands_sequential) class Alignment(data.Text): @@ -369,6 +369,7 @@ class Fasta(Sequence): break # we found a non-empty line, but it's not a fasta header return False + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """Split a FASTA file sequence by sequence. @@ -412,8 +413,8 @@ class Fasta(Sequence): cls._count_split(input_file, batch_size, subdir_generator_function) else: raise Exception('Unsupported split mode %s' % split_params['split_mode']) - split = classmethod(split) + @classmethod def _size_split(cls, input_file, chunk_size, subdir_generator_function): """Split a FASTA file into chunks based on size on disk. @@ -452,8 +453,8 @@ class Fasta(Sequence): f.close() if part_file: part_file.close() - _size_split = classmethod(_size_split) + @classmethod def _count_split(cls, input_file, chunk_size, subdir_generator_function): """Split a FASTA file into chunks based on counting records.""" log.debug("Attemping to split FASTA file %s into chunks of %i sequences" % (input_file, chunk_size)) @@ -489,7 +490,6 @@ class Fasta(Sequence): f.close() if part_file: part_file.close() - _count_split = classmethod(_count_split) @build_sniff_from_prefix @@ -736,6 +736,7 @@ class BaseFastq(Sequence): else: return Sequence.display_data(self, trans, dataset, preview, filename, to_ext, **kwd) + @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): """ FASTQ files are split on cluster boundaries, in increments of 4 lines @@ -759,8 +760,8 @@ class BaseFastq(Sequence): if len(toc_file_datasets) == len(input_datasets): return cls.do_fast_split(input_datasets, toc_file_datasets, subdir_generator_function, split_params) return cls.do_slow_split(input_datasets, subdir_generator_function, split_params) - split = classmethod(split) + @staticmethod def process_split_file(data): """ This is called in the context of an external process launched by a Task (possibly not on the Galaxy machine) @@ -782,7 +783,6 @@ class BaseFastq(Sequence): for cmd in commands: subprocess.check_call(cmd, shell=True) return True - process_split_file = staticmethod(process_split_file) @staticmethod def quality_check(lines): diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 1c56a752b54..efcd99cbfcc 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -51,8 +51,8 @@ class TabularData(data.Text): def set_meta(self, dataset, **kwd): raise NotImplementedError - def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None): - super().set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False) + def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=False, **kwd): + super().set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap) if dataset.metadata.comment_lines: dataset.blurb = "{}, {} comments".format(dataset.blurb, util.commaify(str(dataset.metadata.comment_lines))) @@ -555,6 +555,7 @@ class Sam(Tabular): dataset.metadata.columns = 12 dataset.metadata.column_types = ['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str'] + @staticmethod def merge(split_files, output_file): """ Multiple SAM files may each have headers. Since the headers should all be the same, remove @@ -566,8 +567,6 @@ class Sam(Tabular): cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file] subprocess.check_call(cmd, shell=True) - merge = staticmethod(merge) - # Dataproviders # sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers # TODO:?? seems like there should be an easier way to do this - metadata.comment_char? diff --git a/lib/galaxy/datatypes/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py index 81e852e04bd..5abc068e829 100644 --- a/lib/galaxy/datatypes/util/maf_utilities.py +++ b/lib/galaxy/datatypes/util/maf_utilities.py @@ -17,10 +17,7 @@ import bx.align.maf import bx.interval_index_file import bx.intervals -try: - maketrans = str.maketrans -except AttributeError: - from string import maketrans +maketrans = str.maketrans log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index b28d32b809c..2fbfbe6cfc8 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -58,13 +58,13 @@ class GenericXml(data.Text): """ return file_prefix.startswith(' 1: raise NotImplementedError("Merging multiple XML files is non-trivial and must be implemented for each XML type") # For one file only, use base class method (move/copy) data.Text.merge(split_files, output_file) - merge = staticmethod(merge) @dataproviders.decorators.dataprovider_factory('xml', dataproviders.hierarchy.XMLDataProvider.settings) def xml_dataprovider(self, dataset, **settings): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 502c7eff22d..2f364a95995 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,6 +18,7 @@ import time from collections import defaultdict from datetime import datetime, timedelta from string import Template +from typing import TYPE_CHECKING from uuid import UUID, uuid4 from boltons.iterutils import remap @@ -89,7 +90,17 @@ JOB_METRIC_SCALE = 7 AUTO_PROPAGATED_TAGS = ["name"] -class RepresentById: +if TYPE_CHECKING: + from sqlalchemy.schema import Table + + class _BaseModel: + table: Table = None + +else: + _BaseModel = object + + +class RepresentById(_BaseModel): def __repr__(self): try: r = ''.format(self.__class__.__name__, cached_id(self), hex(id(self))) @@ -231,7 +242,7 @@ class UsesCreateAndUpdateTime: return (galaxy.model.orm.now.now() - create_time).total_seconds() -class WorkerProcess(UsesCreateAndUpdateTime): +class WorkerProcess(UsesCreateAndUpdateTime, _BaseModel): def __init__(self, server_name, hostname): self.server_name = server_name @@ -624,7 +635,7 @@ class User(Dictifiable, RepresentById): session.flush() -class PasswordResetToken: +class PasswordResetToken(_BaseModel): def __init__(self, user, token=None): if token: self.token = token @@ -634,7 +645,7 @@ class PasswordResetToken: self.expiration_time = galaxy.model.orm.now.now() + timedelta(hours=24) -class DynamicTool(Dictifiable): +class DynamicTool(Dictifiable, RepresentById): dict_collection_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') dict_element_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') @@ -2045,7 +2056,7 @@ class History(HasTags, Dictifiable, UsesAnnotations, HasName, RepresentById): rval = 0 return rval - @disk_size.expression + @disk_size.expression # type: ignore def disk_size(cls): """ Return a query scalar that will get any history's size in bytes by summing @@ -2351,7 +2362,7 @@ class StorableObject: sa_session.flush() -class Dataset(StorableObject, RepresentById): +class Dataset(StorableObject, RepresentById, _BaseModel): states = Bunch(NEW='new', UPLOAD='upload', QUEUED='queued', @@ -3494,7 +3505,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot def type_id(self): return '-'.join((self.content_type, str(self.id))) - @type_id.expression + @type_id.expression # type: ignore def type_id(cls): return ((type_coerce(cls.content_type, types.Unicode) + '-' + type_coerce(cls.id, types.Unicode)).label('type_id')) @@ -4410,7 +4421,7 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance, def type_id(self): return '-'.join((self.content_type, str(self.id))) - @type_id.expression + @type_id.expression # type: ignore def type_id(cls): return ((type_coerce(cls.content_type, types.Unicode) + '-' + type_coerce(cls.id, types.Unicode)).label('type_id')) @@ -6104,7 +6115,7 @@ class CustosAuthnzToken(RepresentById): self.refresh_expiration_time = refresh_expiration_time -class CloudAuthz: +class CloudAuthz(_BaseModel): def __init__(self, user_id, provider, config, authn_id, description=""): self.id = None self.user_id = user_id diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index af9ded84c70..257fd9a040d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -2697,7 +2697,7 @@ mapper(model.Visualization, model.Visualization.table, properties=dict( # Set up proxy so that # Visualization.users_shared_with # returns a list of users that visualization is shared with. -model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') +model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore mapper(model.VisualizationUserShareAssociation, model.VisualizationUserShareAssociation.table, properties=dict( user=relation(model.User, @@ -2850,7 +2850,7 @@ def db_next_hid(self, n=1): raise -model.History._next_hid = db_next_hid +model.History._next_hid = db_next_hid # type: ignore def _workflow_invocation_update(self): diff --git a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py index f3d9f6c0c95..45490eac8e9 100644 --- a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py +++ b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py @@ -60,6 +60,8 @@ def directory_hash_id(id): class Dataset: + table: Table = None + states = Bunch(NEW='new', UPLOAD='upload', QUEUED='queued', @@ -338,6 +340,8 @@ class DatasetInstance: class HistoryDatasetAssociation(DatasetInstance): + table: Table = None + def __init__(self, hid=None, history=None, @@ -430,6 +434,8 @@ class HistoryDatasetAssociation(DatasetInstance): class LibraryDatasetDatasetAssociation(DatasetInstance): + table: Table = None + def __init__(self, copied_from_history_dataset_association=None, copied_from_library_dataset_dataset_association=None, @@ -505,6 +511,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance): class LibraryDataset: + table: Table = None + # This class acts as a proxy to the currently selected LDDA def __init__(self, folder=None, order_id=None, name=None, info=None, library_dataset_dataset_association=None, **kwd): self.folder = folder diff --git a/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py b/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py index 99d171ee9e2..4fc497d1600 100644 --- a/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py +++ b/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py @@ -29,6 +29,7 @@ context = scoped_session(sessionmaker(autoflush=False, autocommit=True)) class DeferredJob: + table: Table = None states = Bunch(NEW='new', WAITING='waiting', QUEUED='queued', diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index 78fff7f78ef..5e305d65f89 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -27,6 +27,7 @@ select * from history where name='Unnamed history' import logging import re from json import dumps +from typing import Dict import parsley from sqlalchemy import and_ @@ -89,7 +90,7 @@ class ViewField: class ViewQueryBaseClass: - FIELDS = {} + FIELDS: Dict[str, ViewField] = {} VIEW_NAME = "undefined" def __init__(self): diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index 18ae78b5b5f..a055887d8ed 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -119,7 +119,6 @@ class ModelImportStore(metaclass=abc.ABCMeta): and a history may contain multiple objects with the same 'hid'. """ - @abc.abstractproperty def trust_hid(self, obj_attrs): """Trust HID when importing objects into a new History.""" diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 3f6bf3ad6c8..6d61479b47e 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -12,7 +12,7 @@ from collections import ( namedtuple, OrderedDict ) - +from typing import Any, NamedTuple, Optional import galaxy.model from galaxy import util @@ -697,8 +697,6 @@ def replace_request_syntax_sugar(obj): DiscoveredFile = namedtuple('DiscoveredFile', ['path', 'collector', 'match']) -DiscoveredFileError = namedtuple('DiscoveredFileError', ['error_message', 'collector', 'match']) -DiscoveredFileError.path = None def discovered_file_for_element(dataset, job_working_directory, parent_identifiers=None, collector=None): @@ -828,3 +826,10 @@ class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch): super().__init__( re_match.groupdict(), collector, filename, path=path ) + + +class DiscoveredFileError(NamedTuple): + error_message: str + collector: Any # TODO: setup interface for this + match: JsonCollectedDatasetMatch + path: Optional[str] = None diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index e46b9660141..d69b5e91895 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -1,5 +1,6 @@ import logging import os +from typing import TYPE_CHECKING from galaxy.util import asbool from galaxy.util.bunch import Bunch @@ -9,7 +10,16 @@ from galaxy.util.tool_shed import common_util log = logging.getLogger(__name__) -class ToolShedRepository: +if TYPE_CHECKING: + from sqlalchemy.schema import Table + + class _BaseClass: + table: Table +else: + _BaseClass = object + + +class ToolShedRepository(_BaseClass): dict_collection_visible_keys = ['id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message', 'description'] dict_element_visible_keys = ['id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', @@ -469,20 +479,20 @@ class ToolShedRepository: return False -class RepositoryRepositoryDependencyAssociation: +class RepositoryRepositoryDependencyAssociation(_BaseClass): def __init__(self, tool_shed_repository_id=None, repository_dependency_id=None): self.tool_shed_repository_id = tool_shed_repository_id self.repository_dependency_id = repository_dependency_id -class RepositoryDependency: +class RepositoryDependency(_BaseClass): def __init__(self, tool_shed_repository_id=None): self.tool_shed_repository_id = tool_shed_repository_id -class ToolDependency: +class ToolDependency(_BaseClass): installation_status = Bunch(NEVER_INSTALLED='Never installed', INSTALLING='Installing', INSTALLED='Installed', @@ -569,7 +579,7 @@ class ToolVersion(Dictifiable): return rval -class ToolVersionAssociation: +class ToolVersionAssociation(_BaseClass): def __init__(self, id=None, tool_id=None, parent_id=None): self.id = id @@ -577,7 +587,7 @@ class ToolVersionAssociation: self.parent_id = parent_id -class MigrateTools: +class MigrateTools(_BaseClass): def __init__(self, repository_id=None, repository_path=None, version=None): self.repository_id = repository_id diff --git a/packages/data/galaxy/__init__.py b/packages/data/galaxy/__init__.py index 69e3be50dac..0d1f7edf5dc 100644 --- a/packages/data/galaxy/__init__.py +++ b/packages/data/galaxy/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/packages/test.sh b/packages/test.sh index ff7882f7f6a..e2b476e47b7 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -34,7 +34,7 @@ PACKAGE_DIRS=( # tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. RUN_TESTS=(1 1 1 1 1 1 1 1 1 1 0 0) -RUN_MYPY=(1 1 1 1 1 0 0 0 0 0 0 0) +RUN_MYPY=(1 1 1 1 1 1 0 0 0 0 0 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} run_tests=${RUN_TESTS[$i]} diff --git a/test/unit/data/datatypes/dataproviders/test_base_dataproviders.py b/test/unit/data/datatypes/dataproviders/test_base_dataproviders.py index 228938acdf6..3ccd7af0b06 100644 --- a/test/unit/data/datatypes/dataproviders/test_base_dataproviders.py +++ b/test/unit/data/datatypes/dataproviders/test_base_dataproviders.py @@ -7,6 +7,7 @@ import os import tempfile import unittest from io import StringIO +from typing import Type from galaxy.datatypes.dataproviders import base, exceptions from galaxy.util import clean_multiline_string @@ -57,7 +58,7 @@ class BaseTestCase(unittest.TestCase): class Test_BaseDataProvider(BaseTestCase): - provider_class = base.DataProvider + provider_class: Type[base.DataProvider] = base.DataProvider def contents_provider_and_data(self, filename=None, contents=None, source=None, *provider_args, **provider_kwargs): @@ -156,7 +157,7 @@ class Test_BaseDataProvider(BaseTestCase): class Test_FilteredDataProvider(Test_BaseDataProvider): - provider_class = base.FilteredDataProvider + provider_class: Type[base.DataProvider] = base.FilteredDataProvider def assertCounters(self, provider, read, valid, returned): self.assertEqual(provider.num_data_read, read) @@ -182,7 +183,7 @@ class Test_FilteredDataProvider(Test_BaseDataProvider): class Test_LimitedOffsetDataProvider(Test_FilteredDataProvider): - provider_class = base.LimitedOffsetDataProvider + provider_class: Type[base.DataProvider] = base.LimitedOffsetDataProvider def test_offset_1(self): """when offset is 1, should skip first @@ -289,7 +290,7 @@ class Test_LimitedOffsetDataProvider(Test_FilteredDataProvider): class Test_MultiSourceDataProvider(BaseTestCase): - provider_class = base.MultiSourceDataProvider + provider_class: Type[base.DataProvider] = base.MultiSourceDataProvider def contents_and_tmpfile(self, contents=None): # TODO: hmmmm... diff --git a/test/unit/data/datatypes/dataproviders/test_line_dataproviders.py b/test/unit/data/datatypes/dataproviders/test_line_dataproviders.py index c785f120c1b..60b6cfdee7e 100644 --- a/test/unit/data/datatypes/dataproviders/test_line_dataproviders.py +++ b/test/unit/data/datatypes/dataproviders/test_line_dataproviders.py @@ -4,7 +4,9 @@ Unit tests for base DataProviders. """ import logging import unittest +from typing import Type +from galaxy.datatypes.dataproviders import base from galaxy.datatypes.dataproviders import line from . import test_base_dataproviders @@ -13,7 +15,7 @@ log = logging.getLogger(__name__) # TODO: TestCase hierarchy is a bit of mess here. class Test_FilteredLineDataProvider(test_base_dataproviders.Test_FilteredDataProvider): - provider_class = line.FilteredLineDataProvider + provider_class: Type[base.DataProvider] = line.FilteredLineDataProvider default_file_contents = """ # this should be stripped out One