opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11007 from jmchilton/mypy_galaxy_data

Browse Source 
Type checking fixes for galaxy-data (datatypes and models).
This commit is contained in:
Björn Grüning
2020-12-29 21:11:29 +01:00
committed by GitHub
parent 4ba40970d2 a7b6f08446
commit cf5ee5aed0
26 changed files with 118 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/anvio.py
+2 -1
View File
@@ -6,6 +6,7 @@ import glob
import logging
import os
import sys
from typing import Optional
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.text import Html
@@ -79,7 +80,7 @@ class AnvioComposite(Html):
class AnvioDB(AnvioComposite):
"""Class for AnvioDB database files."""
_anvio_basename = None
_anvio_basename: Optional[str] = None
MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True)
file_ext = 'anvio_db'
lib/galaxy/datatypes/binary.py
+4 -3
View File
@@ -14,6 +14,7 @@ import tempfile
import zipfile
from collections import OrderedDict
from json import dumps
from typing import Optional
import h5py
import pysam
@@ -51,7 +52,7 @@ class Binary(data.Data):
def register_unsniffable_binary_ext(ext):
"""Deprecated method."""
def set_peek(self, dataset, is_multi_byte=False):
def set_peek(self, dataset, **kwd):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = 'binary data'
@@ -272,7 +273,7 @@ class BamNative(CompressedArchive):
edam_format = "format_2572"
edam_data = "data_0863"
file_ext = "unsorted.bam"
sort_flag = None
sort_flag: Optional[str] = None
MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None)
MetadataElement(name="sort_order", default=None, desc="Sort Order", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None)
@@ -1122,7 +1123,7 @@ class GmxBinary(Binary):
Base class for GROMACS binary files - xtc, trr, cpt
"""
magic_number = None # variables to be overwritten in the child class
magic_number: Optional[int] = None # variables to be overwritten in the child class
file_ext = ""
def sniff(self, filename):
lib/galaxy/datatypes/blast.py
+1 -1
View File
@@ -89,6 +89,7 @@ class BlastXml(GenericXml):
return False
return True
@staticmethod
def merge(split_files, output_file):
"""Merging multiple XML files is non-trivial and must be done in subclasses."""
if len(split_files) == 1:
@@ -167,7 +168,6 @@ class BlastXml(GenericXml):
h.close()
out.write(" </BlastOutput_iterations>\n")
out.write("</BlastOutput>\n")
merge = staticmethod(merge)
class _BlastDb(Data):
lib/galaxy/datatypes/constructive_solid_geometry.py
+4
View File
@@ -1,6 +1,10 @@
# TODO: revisit ignoring type and write some tests for this, the multi-inheritance in this
# this file is challenging, it should be broken into true mixins.
# type: ignore
"""
Constructive Solid Geometry file formats.
"""
import abc
from galaxy import util
lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py
-2
View File
@@ -13,8 +13,6 @@ Output: tabular
import os
import sys
seq_hash = {}
def __main__():
infile = sys.argv[1]
lib/galaxy/datatypes/data.py
+13 -13
View File
@@ -7,6 +7,7 @@ import string
import tempfile
from collections import OrderedDict
from inspect import isclass
from typing import Any, Dict, Optional
import webob.exc
from markupsafe import escape
@@ -116,27 +117,27 @@ class Data(metaclass=DataMeta):
# Add metadata elements
MetadataElement(name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, no_value="?")
# Stores the set of display applications, and viewing methods, supported by this datatype
supported_display_apps = {}
supported_display_apps: Dict[str, Any] = {}
# If False, the peek is regenerated whenever a dataset of this type is copied
copy_safe_peek = True
# The dataset contains binary data --> do not space_to_tab or convert newlines, etc.
# Allow binary file uploads of this type when True.
is_binary = True
# Composite datatypes
composite_type = None
composite_files = OrderedDict()
composite_type: Optional[str] = None
composite_files: Dict[str, Any] = OrderedDict()
primary_file_name = 'index'
# Allow user to change between this datatype and others. If left to None,
# datatype change is allowed if the datatype is not composite.
allow_datatype_change = None
allow_datatype_change: Optional[bool] = None
# A per datatype setting (inherited): max file size (in bytes) for setting optional metadata
_max_optional_metadata_filesize = None
# Trackster track type.
track_type = None
track_type: Optional[str] = None
# Data sources.
data_sources = {}
data_sources: Dict[str, str] = {}
def __init__(self, **kwd):
"""Initialize the datatype"""
@@ -180,7 +181,7 @@ class Data(metaclass=DataMeta):
if copy_from:
dataset.metadata = copy_from.metadata
def set_meta(self, dataset, overwrite=True, **kwd):
def set_meta(self, dataset: Any, overwrite=True, **kwd):
"""Unimplemented method, allows guessing of metadata from contents of file"""
return True
@@ -681,11 +682,11 @@ class Data(metaclass=DataMeta):
return key
@property
def writable_files(self, dataset=None):
def writable_files(self):
files = OrderedDict()
if self.composite_type != 'auto_primary_file':
files[self.primary_file_name] = self.__new_composite_file(self.primary_file_name)
for key, value in self.get_composite_files(dataset=dataset).items():
for key, value in self.get_composite_files().items():
files[key] = value
return files
@@ -718,6 +719,7 @@ class Data(metaclass=DataMeta):
datatype_classes = tuple(datatype if isclass(datatype) else datatype.__class__ for datatype in target_datatypes)
return isinstance(self, datatype_classes)
@staticmethod
def merge(split_files, output_file):
"""
Merge files with copy.copyfileobj() will not hit the
@@ -732,8 +734,6 @@ class Data(metaclass=DataMeta):
for fsrc in split_files:
shutil.copyfileobj(open(fsrc, 'rb'), fdst)
merge = staticmethod(merge)
def get_visualizations(self, dataset):
"""
Returns a list of visualizations for datatype.
@@ -844,7 +844,7 @@ class Text(Data):
data_lines = None
return data_lines
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=True):
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=True, **kwd):
"""
Set the peek. This method is used by various subclasses of Text.
"""
@@ -879,6 +879,7 @@ class Text(Data):
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
Split the input files by line.
@@ -947,7 +948,6 @@ class Text(Data):
f.close()
if part_file:
part_file.close()
split = classmethod(split)
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory('line', dataproviders.line.FilteredLineDataProvider.settings)
lib/galaxy/datatypes/dataproviders/base.py
+2 -2
View File
@@ -9,7 +9,7 @@ Base class(es) for all DataProviders.
import logging
from collections import deque
from typing import Dict
from . import exceptions
@@ -74,7 +74,7 @@ class DataProvider(metaclass=HasSettings):
# a definition of expected types for keyword arguments sent to __init__
# useful for controlling how query string dictionaries can be parsed into correct types for __init__
# empty in this base class
settings = {}
settings: Dict[str, str] = {}
def __init__(self, source, **kwargs):
"""
lib/galaxy/datatypes/display_applications/parameters.py
+2 -1
View File
@@ -1,5 +1,6 @@
# Contains parameters that are used in Display Applications
import mimetypes
from typing import Optional
from urllib.parse import quote_plus
from galaxy.util import string_as_bool
@@ -12,7 +13,7 @@ DEFAULT_DATASET_NAME = 'dataset'
class DisplayApplicationParameter:
""" Abstract Class for Display Application Parameters """
type = None
type: Optional[str] = None
@classmethod
def from_elem(cls, elem, link):
lib/galaxy/datatypes/molecules.py
+6 -6
View File
@@ -125,6 +125,7 @@ class SDF(GenericMolFile):
"""
dataset.metadata.number_of_molecules = count_special_lines(r"^\$\$\$\$$", dataset.file_name)
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
Split the input files by molecule records.
@@ -172,7 +173,6 @@ class SDF(GenericMolFile):
except Exception as e:
log.error('Unable to split files: %s', unicodify(e))
raise
split = classmethod(split)
@build_sniff_from_prefix
@@ -208,6 +208,7 @@ class MOL2(GenericMolFile):
"""
dataset.metadata.number_of_molecules = count_special_lines("@<TRIPOS>MOLECULE", dataset.file_name)
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
Split the input files by molecule records.
@@ -259,7 +260,6 @@ class MOL2(GenericMolFile):
except Exception as e:
log.error('Unable to split files: %s', unicodify(e))
raise
split = classmethod(split)
@build_sniff_from_prefix
@@ -293,6 +293,7 @@ class FPS(GenericMolFile):
"""
dataset.metadata.number_of_molecules = count_special_lines('^#', dataset.file_name, invert=True)
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
Split the input files by fingerprint records.
@@ -338,8 +339,8 @@ class FPS(GenericMolFile):
except Exception as e:
log.error('Unable to split files: %s', unicodify(e))
raise
split = classmethod(split)
@staticmethod
def merge(split_files, output_file):
"""
Merging fps files requires merging the header manually.
@@ -363,7 +364,6 @@ class FPS(GenericMolFile):
# line is no header and not a comment, we assume the first header is written to out and we set 'first' to False
first = False
out.write(line)
merge = staticmethod(merge)
class OBFS(Binary):
@@ -846,6 +846,7 @@ class CML(GenericXml):
return True
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
Split the input files by molecule records.
@@ -902,8 +903,8 @@ class CML(GenericXml):
except Exception as e:
log.error('Unable to split files: %s', unicodify(e))
raise
split = classmethod(split)
@staticmethod
def merge(split_files, output_file):
"""
Merging CML files.
@@ -938,4 +939,3 @@ class CML(GenericXml):
if molecule_found:
out.write(line)
out.write("</cml>\n")
merge = staticmethod(merge)
lib/galaxy/datatypes/msa.py
+1 -1
View File
@@ -169,6 +169,7 @@ class Stockholm_1_0(Text):
"""
dataset.metadata.number_of_models = generic_util.count_special_lines('^#[[:space:]+]STOCKHOLM[[:space:]+]1.0', dataset.file_name)
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
@@ -218,7 +219,6 @@ class Stockholm_1_0(Text):
except Exception as e:
log.error('Unable to split files: %s', unicodify(e))
raise
split = classmethod(split)
@build_sniff_from_prefix
lib/galaxy/datatypes/sequence.py
+11 -11
View File
@@ -120,6 +120,7 @@ class Sequence(data.Text):
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@staticmethod
def get_sequences_per_file(total_sequences, split_params):
if split_params['split_mode'] == 'number_of_parts':
# legacy basic mode - split into a specified number of parts
@@ -138,8 +139,8 @@ class Sequence(data.Text):
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
return sequences_per_file
get_sequences_per_file = staticmethod(get_sequences_per_file)
@classmethod
def do_slow_split(cls, input_datasets, subdir_generator_function, split_params):
# count the sequences so we can split
# TODO: if metadata is present, take the number of lines / 4
@@ -152,8 +153,8 @@ class Sequence(data.Text):
sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params)
return cls.write_split_files(input_datasets, None, subdir_generator_function, sequences_per_file)
do_slow_split = classmethod(do_slow_split)
@classmethod
def do_fast_split(cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params):
data = json.load(open(toc_file_datasets[0].file_name))
sections = data['sections']
@@ -162,8 +163,8 @@ class Sequence(data.Text):
total_sequences += int(section['sequences'])
sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params)
return cls.write_split_files(input_datasets, toc_file_datasets, subdir_generator_function, sequences_per_file)
do_fast_split = classmethod(do_fast_split)
@classmethod
def write_split_files(cls, input_datasets, toc_file_datasets, subdir_generator_function, sequences_per_file):
directories = []
@@ -195,7 +196,6 @@ class Sequence(data.Text):
json.dump(split_data, f)
start_sequence += sequences_per_file[part_no]
return directories
write_split_files = classmethod(write_split_files)
def split(cls, input_datasets, subdir_generator_function, split_params):
"""Split a generic sequence file (not sensible or possible, see subclasses)."""
@@ -203,6 +203,7 @@ class Sequence(data.Text):
return None
raise NotImplementedError("Can't split generic sequence files")
@staticmethod
def get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count):
"""
Uses a Table of Contents dict, parsed from an FQTOC file, to come up with a set of
@@ -270,8 +271,8 @@ class Sequence(data.Text):
raise Exception('%s sequences not found in file' % sequence_count)
return result
get_split_commands_with_toc = staticmethod(get_split_commands_with_toc)
@staticmethod
def get_split_commands_sequential(is_compressed, input_name, output_name, start_sequence, sequence_count):
"""
Does a brain-dead sequential scan & extract of certain sequences
@@ -290,7 +291,6 @@ class Sequence(data.Text):
cmd += ' > "%s"' % output_name
return [cmd]
get_split_commands_sequential = staticmethod(get_split_commands_sequential)
class Alignment(data.Text):
@@ -369,6 +369,7 @@ class Fasta(Sequence):
break # we found a non-empty line, but it's not a fasta header
return False
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""Split a FASTA file sequence by sequence.
@@ -412,8 +413,8 @@ class Fasta(Sequence):
cls._count_split(input_file, batch_size, subdir_generator_function)
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
split = classmethod(split)
@classmethod
def _size_split(cls, input_file, chunk_size, subdir_generator_function):
"""Split a FASTA file into chunks based on size on disk.
@@ -452,8 +453,8 @@ class Fasta(Sequence):
f.close()
if part_file:
part_file.close()
_size_split = classmethod(_size_split)
@classmethod
def _count_split(cls, input_file, chunk_size, subdir_generator_function):
"""Split a FASTA file into chunks based on counting records."""
log.debug("Attemping to split FASTA file %s into chunks of %i sequences" % (input_file, chunk_size))
@@ -489,7 +490,6 @@ class Fasta(Sequence):
f.close()
if part_file:
part_file.close()
_count_split = classmethod(_count_split)
@build_sniff_from_prefix
@@ -736,6 +736,7 @@ class BaseFastq(Sequence):
else:
return Sequence.display_data(self, trans, dataset, preview, filename, to_ext, **kwd)
@classmethod
def split(cls, input_datasets, subdir_generator_function, split_params):
"""
FASTQ files are split on cluster boundaries, in increments of 4 lines
@@ -759,8 +760,8 @@ class BaseFastq(Sequence):
if len(toc_file_datasets) == len(input_datasets):
return cls.do_fast_split(input_datasets, toc_file_datasets, subdir_generator_function, split_params)
return cls.do_slow_split(input_datasets, subdir_generator_function, split_params)
split = classmethod(split)
@staticmethod
def process_split_file(data):
"""
This is called in the context of an external process launched by a Task (possibly not on the Galaxy machine)
@@ -782,7 +783,6 @@ class BaseFastq(Sequence):
for cmd in commands:
subprocess.check_call(cmd, shell=True)
return True
process_split_file = staticmethod(process_split_file)
@staticmethod
def quality_check(lines):
lib/galaxy/datatypes/tabular.py
+3 -4
View File
@@ -51,8 +51,8 @@ class TabularData(data.Text):
def set_meta(self, dataset, **kwd):
raise NotImplementedError
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None):
super().set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False)
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=False, **kwd):
super().set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap)
if dataset.metadata.comment_lines:
dataset.blurb = "{}, {} comments".format(dataset.blurb, util.commaify(str(dataset.metadata.comment_lines)))
@@ -555,6 +555,7 @@ class Sam(Tabular):
dataset.metadata.columns = 12
dataset.metadata.column_types = ['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str']
@staticmethod
def merge(split_files, output_file):
"""
Multiple SAM files may each have headers. Since the headers should all be the same, remove
@@ -566,8 +567,6 @@ class Sam(Tabular):
cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file]
subprocess.check_call(cmd, shell=True)
merge = staticmethod(merge)
# Dataproviders
# sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers
# TODO:?? seems like there should be an easier way to do this - metadata.comment_char?
lib/galaxy/datatypes/util/maf_utilities.py
+1 -4
View File
@@ -17,10 +17,7 @@ import bx.align.maf
import bx.interval_index_file
import bx.intervals
try:
maketrans = str.maketrans
except AttributeError:
from string import maketrans
maketrans = str.maketrans
log = logging.getLogger(__name__)
lib/galaxy/datatypes/xml.py
+1 -1
View File
@@ -58,13 +58,13 @@ class GenericXml(data.Text):
"""
return file_prefix.startswith('<?xml ')
@staticmethod
def merge(split_files, output_file):
"""Merging multiple XML files is non-trivial and must be done in subclasses."""
if len(split_files) > 1:
raise NotImplementedError("Merging multiple XML files is non-trivial and must be implemented for each XML type")
# For one file only, use base class method (move/copy)
data.Text.merge(split_files, output_file)
merge = staticmethod(merge)
@dataproviders.decorators.dataprovider_factory('xml', dataproviders.hierarchy.XMLDataProvider.settings)
def xml_dataprovider(self, dataset, **settings):
lib/galaxy/model/__init__.py
+20 -9
View File
@@ -18,6 +18,7 @@ import time
from collections import defaultdict
from datetime import datetime, timedelta
from string import Template
from typing import TYPE_CHECKING
from uuid import UUID, uuid4
from boltons.iterutils import remap
@@ -89,7 +90,17 @@ JOB_METRIC_SCALE = 7
AUTO_PROPAGATED_TAGS = ["name"]
class RepresentById:
if TYPE_CHECKING:
from sqlalchemy.schema import Table
class _BaseModel:
table: Table = None
else:
_BaseModel = object
class RepresentById(_BaseModel):
def __repr__(self):
try:
r = '<galaxy.model.{}({}) at {}>'.format(self.__class__.__name__, cached_id(self), hex(id(self)))
@@ -231,7 +242,7 @@ class UsesCreateAndUpdateTime:
return (galaxy.model.orm.now.now() - create_time).total_seconds()
class WorkerProcess(UsesCreateAndUpdateTime):
class WorkerProcess(UsesCreateAndUpdateTime, _BaseModel):
def __init__(self, server_name, hostname):
self.server_name = server_name
@@ -624,7 +635,7 @@ class User(Dictifiable, RepresentById):
session.flush()
class PasswordResetToken:
class PasswordResetToken(_BaseModel):
def __init__(self, user, token=None):
if token:
self.token = token
@@ -634,7 +645,7 @@ class PasswordResetToken:
self.expiration_time = galaxy.model.orm.now.now() + timedelta(hours=24)
class DynamicTool(Dictifiable):
class DynamicTool(Dictifiable, RepresentById):
dict_collection_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden')
dict_element_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden')
@@ -2045,7 +2056,7 @@ class History(HasTags, Dictifiable, UsesAnnotations, HasName, RepresentById):
rval = 0
return rval
@disk_size.expression
@disk_size.expression # type: ignore
def disk_size(cls):
"""
Return a query scalar that will get any history's size in bytes by summing
@@ -2351,7 +2362,7 @@ class StorableObject:
sa_session.flush()
class Dataset(StorableObject, RepresentById):
class Dataset(StorableObject, RepresentById, _BaseModel):
states = Bunch(NEW='new',
UPLOAD='upload',
QUEUED='queued',
@@ -3494,7 +3505,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
def type_id(self):
return '-'.join((self.content_type, str(self.id)))
@type_id.expression
@type_id.expression # type: ignore
def type_id(cls):
return ((type_coerce(cls.content_type, types.Unicode) + '-'
+ type_coerce(cls.id, types.Unicode)).label('type_id'))
@@ -4410,7 +4421,7 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance,
def type_id(self):
return '-'.join((self.content_type, str(self.id)))
@type_id.expression
@type_id.expression # type: ignore
def type_id(cls):
return ((type_coerce(cls.content_type, types.Unicode) + '-'
+ type_coerce(cls.id, types.Unicode)).label('type_id'))
@@ -6104,7 +6115,7 @@ class CustosAuthnzToken(RepresentById):
self.refresh_expiration_time = refresh_expiration_time
class CloudAuthz:
class CloudAuthz(_BaseModel):
def __init__(self, user_id, provider, config, authn_id, description=""):
self.id = None
self.user_id = user_id
lib/galaxy/model/mapping.py
+2 -2
View File
@@ -2697,7 +2697,7 @@ mapper(model.Visualization, model.Visualization.table, properties=dict(
# Set up proxy so that
# Visualization.users_shared_with
# returns a list of users that visualization is shared with.
model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user')
model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore
mapper(model.VisualizationUserShareAssociation, model.VisualizationUserShareAssociation.table, properties=dict(
user=relation(model.User,
@@ -2850,7 +2850,7 @@ def db_next_hid(self, n=1):
raise
model.History._next_hid = db_next_hid
model.History._next_hid = db_next_hid # type: ignore
def _workflow_invocation_update(self):
lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py
+8
View File
@@ -60,6 +60,8 @@ def directory_hash_id(id):
class Dataset:
table: Table = None
states = Bunch(NEW='new',
UPLOAD='upload',
QUEUED='queued',
@@ -338,6 +340,8 @@ class DatasetInstance:
class HistoryDatasetAssociation(DatasetInstance):
table: Table = None
def __init__(self,
hid=None,
history=None,
@@ -430,6 +434,8 @@ class HistoryDatasetAssociation(DatasetInstance):
class LibraryDatasetDatasetAssociation(DatasetInstance):
table: Table = None
def __init__(self,
copied_from_history_dataset_association=None,
copied_from_library_dataset_dataset_association=None,
@@ -505,6 +511,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance):
class LibraryDataset:
table: Table = None
# This class acts as a proxy to the currently selected LDDA
def __init__(self, folder=None, order_id=None, name=None, info=None, library_dataset_dataset_association=None, **kwd):
self.folder = folder
lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py
+1
View File
@@ -29,6 +29,7 @@ context = scoped_session(sessionmaker(autoflush=False, autocommit=True))
class DeferredJob:
table: Table = None
states = Bunch(NEW='new',
WAITING='waiting',
QUEUED='queued',
lib/galaxy/model/search.py
+2 -1
View File
@@ -27,6 +27,7 @@ select * from history where name='Unnamed history'
import logging
import re
from json import dumps
from typing import Dict
import parsley
from sqlalchemy import and_
@@ -89,7 +90,7 @@ class ViewField:
class ViewQueryBaseClass:
FIELDS = {}
FIELDS: Dict[str, ViewField] = {}
VIEW_NAME = "undefined"
def __init__(self):
lib/galaxy/model/store/__init__.py
-1
View File
@@ -119,7 +119,6 @@ class ModelImportStore(metaclass=abc.ABCMeta):
and a history may contain multiple objects with the same 'hid'.
"""
@abc.abstractproperty
def trust_hid(self, obj_attrs):
"""Trust HID when importing objects into a new History."""
lib/galaxy/model/store/discover.py
+8 -3
View File
@@ -12,7 +12,7 @@ from collections import (
namedtuple,
OrderedDict
)
from typing import Any, NamedTuple, Optional
import galaxy.model
from galaxy import util
@@ -697,8 +697,6 @@ def replace_request_syntax_sugar(obj):
DiscoveredFile = namedtuple('DiscoveredFile', ['path', 'collector', 'match'])
DiscoveredFileError = namedtuple('DiscoveredFileError', ['error_message', 'collector', 'match'])
DiscoveredFileError.path = None
def discovered_file_for_element(dataset, job_working_directory, parent_identifiers=None, collector=None):
@@ -828,3 +826,10 @@ class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
super().__init__(
re_match.groupdict(), collector, filename, path=path
)
class DiscoveredFileError(NamedTuple):
error_message: str
collector: Any # TODO: setup interface for this
match: JsonCollectedDatasetMatch
path: Optional[str] = None
lib/galaxy/model/tool_shed_install/__init__.py
+16 -6
View File
@@ -1,5 +1,6 @@
import logging
import os
from typing import TYPE_CHECKING
from galaxy.util import asbool
from galaxy.util.bunch import Bunch
@@ -9,7 +10,16 @@ from galaxy.util.tool_shed import common_util
log = logging.getLogger(__name__)
class ToolShedRepository:
if TYPE_CHECKING:
from sqlalchemy.schema import Table
class _BaseClass:
table: Table
else:
_BaseClass = object
class ToolShedRepository(_BaseClass):
dict_collection_visible_keys = ['id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes',
'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message', 'description']
dict_element_visible_keys = ['id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes',
@@ -469,20 +479,20 @@ class ToolShedRepository:
return False
class RepositoryRepositoryDependencyAssociation:
class RepositoryRepositoryDependencyAssociation(_BaseClass):
def __init__(self, tool_shed_repository_id=None, repository_dependency_id=None):
self.tool_shed_repository_id = tool_shed_repository_id
self.repository_dependency_id = repository_dependency_id
class RepositoryDependency:
class RepositoryDependency(_BaseClass):
def __init__(self, tool_shed_repository_id=None):
self.tool_shed_repository_id = tool_shed_repository_id
class ToolDependency:
class ToolDependency(_BaseClass):
installation_status = Bunch(NEVER_INSTALLED='Never installed',
INSTALLING='Installing',
INSTALLED='Installed',
@@ -569,7 +579,7 @@ class ToolVersion(Dictifiable):
return rval
class ToolVersionAssociation:
class ToolVersionAssociation(_BaseClass):
def __init__(self, id=None, tool_id=None, parent_id=None):
self.id = id
@@ -577,7 +587,7 @@ class ToolVersionAssociation:
self.parent_id = parent_id
class MigrateTools:
class MigrateTools(_BaseClass):
def __init__(self, repository_id=None, repository_path=None, version=None):
self.repository_id = repository_id
packages/data/galaxy/__init__.py
+1 -1
View File
@@ -1 +1 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore
packages/test.sh
+1 -1
View File
@@ -34,7 +34,7 @@ PACKAGE_DIRS=(
# tool_util not yet working 100%,
# data has many problems quota, tool shed install database, etc..
RUN_TESTS=(1 1 1 1 1 1 1 1 1 1 0 0)
RUN_MYPY=(1 1 1 1 1 0 0 0 0 0 0 0)
RUN_MYPY=(1 1 1 1 1 1 0 0 0 0 0 0)
for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do
package_dir=${PACKAGE_DIRS[$i]}
run_tests=${RUN_TESTS[$i]}
test/unit/data/datatypes/dataproviders/test_base_dataproviders.py
+5 -4
View File
@@ -7,6 +7,7 @@ import os
import tempfile
import unittest
from io import StringIO
from typing import Type
from galaxy.datatypes.dataproviders import base, exceptions
from galaxy.util import clean_multiline_string
@@ -57,7 +58,7 @@ class BaseTestCase(unittest.TestCase):
class Test_BaseDataProvider(BaseTestCase):
provider_class = base.DataProvider
provider_class: Type[base.DataProvider] = base.DataProvider
def contents_provider_and_data(self,
filename=None, contents=None, source=None, *provider_args, **provider_kwargs):
@@ -156,7 +157,7 @@ class Test_BaseDataProvider(BaseTestCase):
class Test_FilteredDataProvider(Test_BaseDataProvider):
provider_class = base.FilteredDataProvider
provider_class: Type[base.DataProvider] = base.FilteredDataProvider
def assertCounters(self, provider, read, valid, returned):
self.assertEqual(provider.num_data_read, read)
@@ -182,7 +183,7 @@ class Test_FilteredDataProvider(Test_BaseDataProvider):
class Test_LimitedOffsetDataProvider(Test_FilteredDataProvider):
provider_class = base.LimitedOffsetDataProvider
provider_class: Type[base.DataProvider] = base.LimitedOffsetDataProvider
def test_offset_1(self):
"""when offset is 1, should skip first
@@ -289,7 +290,7 @@ class Test_LimitedOffsetDataProvider(Test_FilteredDataProvider):
class Test_MultiSourceDataProvider(BaseTestCase):
provider_class = base.MultiSourceDataProvider
provider_class: Type[base.DataProvider] = base.MultiSourceDataProvider
def contents_and_tmpfile(self, contents=None):
# TODO: hmmmm...
test/unit/data/datatypes/dataproviders/test_line_dataproviders.py
+3 -1
View File
@@ -4,7 +4,9 @@ Unit tests for base DataProviders.
"""
import logging
import unittest
from typing import Type
from galaxy.datatypes.dataproviders import base
from galaxy.datatypes.dataproviders import line
from . import test_base_dataproviders
@@ -13,7 +15,7 @@ log = logging.getLogger(__name__)
# TODO: TestCase hierarchy is a bit of mess here.
class Test_FilteredLineDataProvider(test_base_dataproviders.Test_FilteredDataProvider):
provider_class = line.FilteredLineDataProvider
provider_class: Type[base.DataProvider] = line.FilteredLineDataProvider
default_file_contents = """
# this should be stripped out
One