diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 58926ca2e3a..d0c2fe1eb9e 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -24,10 +24,8 @@ from galaxy.util import ( from galaxy.util.bunch import Bunch from galaxy.util.sanitize_html import sanitize_html from galaxy.util.zipstream import ZipstreamWrapper -from . import ( - dataproviders, - metadata -) +from . import dataproviders as p_dataproviders +from . import metadata if TYPE_CHECKING: from galaxy.model import DatasetInstance @@ -114,7 +112,7 @@ class DataMeta(abc.ABCMeta): metadata.Statement.process(cls) -@dataproviders.decorators.has_dataproviders +@p_dataproviders.decorators.has_dataproviders class Data(metaclass=DataMeta): """ Base class for all datatypes. Implements basic interfaces as well @@ -137,7 +135,7 @@ class Data(metaclass=DataMeta): CHUNKABLE = False #: Dictionary of metadata fields for this datatype - metadata_spec = None + metadata_spec: metadata.MetadataSpecCollection # Add metadata elements MetadataElement(name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, no_value="?") @@ -164,9 +162,10 @@ class Data(metaclass=DataMeta): # Data sources. data_sources: Dict[str, str] = {} + dataproviders: Dict[str, Any] + def __init__(self, **kwd): """Initialize the datatype""" - object.__init__(self, **kwd) self.supported_display_apps = self.supported_display_apps.copy() self.composite_files = self.composite_files.copy() self.display_applications = {} @@ -272,10 +271,9 @@ class Data(metaclass=DataMeta): continue out.append(f"{escape(unicodify(line, 'utf-8'))}") out.append('') - out = "".join(out) + return "".join(out) except Exception as exc: - out = f"Can't create peek: {unicodify(exc)}" - return out + return f"Can't create peek: {unicodify(exc)}" def _archive_main_file(self, archive, display_name, data_filename): """Called from _archive_composite_dataset to add central file to archive. @@ -428,8 +426,15 @@ class Data(metaclass=DataMeta): self._clean_and_set_mime_type(trans, data.get_mime()) trans.log_event(f"Display dataset id: {str(data.id)}") - from galaxy import datatypes # DBTODO REMOVE THIS AT REFACTOR - if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file + from galaxy.datatypes import ( + binary, + images, + text, + ) # DBTODO REMOVE THIS AT REFACTOR + + if to_ext or isinstance( + data.datatype, binary.Binary + ): # Saving the file, or binary file if data.extension in composite_extensions: return self._archive_composite_dataset(trans, data, do_action=kwd.get('do_action', 'zip')) else: @@ -441,10 +446,14 @@ class Data(metaclass=DataMeta): if not os.path.exists(data.file_name): raise webob.exc.HTTPNotFound(f"File Not Found ({data.file_name}).") max_peek_size = DEFAULT_MAX_PEEK_SIZE # 1 MB - if isinstance(data.datatype, datatypes.text.Html): + if isinstance(data.datatype, text.Html): max_peek_size = 10000000 # 10 MB for html preview = util.string_as_bool(preview) - if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat(data.file_name).st_size < max_peek_size: + if ( + not preview + or isinstance(data.datatype, images.Image) + or os.stat(data.file_name).st_size < max_peek_size + ): return self._yield_user_file_content(trans, data, data.file_name) else: trans.response.set_content_type("text/html") @@ -537,7 +546,7 @@ class Data(metaclass=DataMeta): """Returns formatted html of dataset info""" try: # Change new line chars to html - info = escape(dataset.info) + info: str = escape(dataset.info) if info.find('\r\n') >= 0: info = info.replace('\r\n', '
') if info.find('\r') >= 0: @@ -697,7 +706,7 @@ class Data(metaclass=DataMeta): if dataset: meta_value = str(dataset.metadata.get(composite_file.substitute_name_with_metadata)) else: - meta_value = self.spec[composite_file.substitute_name_with_metadata].default + meta_value = self.spec[composite_file.substitute_name_with_metadata].default # type: ignore return key % meta_value return key @@ -777,25 +786,29 @@ class Data(metaclass=DataMeta): """ if self.has_dataprovider(data_format): return self.dataproviders[data_format](self, dataset, **settings) - raise dataproviders.exceptions.NoProviderAvailable(self, data_format) + raise p_dataproviders.exceptions.NoProviderAvailable(self, data_format) def validate(self, dataset, **kwd): return DatatypeValidation.unvalidated() - @dataproviders.decorators.dataprovider_factory('base') + @p_dataproviders.decorators.dataprovider_factory("base") def base_dataprovider(self, dataset, **settings): - dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - return dataproviders.base.DataProvider(dataset_source, **settings) + dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) + return p_dataproviders.base.DataProvider(dataset_source, **settings) - @dataproviders.decorators.dataprovider_factory('chunk', dataproviders.chunk.ChunkDataProvider.settings) + @p_dataproviders.decorators.dataprovider_factory( + "chunk", p_dataproviders.chunk.ChunkDataProvider.settings + ) def chunk_dataprovider(self, dataset, **settings): - dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - return dataproviders.chunk.ChunkDataProvider(dataset_source, **settings) + dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) + return p_dataproviders.chunk.ChunkDataProvider(dataset_source, **settings) - @dataproviders.decorators.dataprovider_factory('chunk64', dataproviders.chunk.Base64ChunkDataProvider.settings) + @p_dataproviders.decorators.dataprovider_factory( + "chunk64", p_dataproviders.chunk.Base64ChunkDataProvider.settings + ) def chunk64_dataprovider(self, dataset, **settings): - dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - return dataproviders.chunk.Base64ChunkDataProvider(dataset_source, **settings) + dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) + return p_dataproviders.chunk.Base64ChunkDataProvider(dataset_source, **settings) def _clean_and_set_mime_type(self, trans, mime): if mime.lower() in XSS_VULNERABLE_MIME_TYPES: @@ -803,8 +816,11 @@ class Data(metaclass=DataMeta): mime = DEFAULT_MIME_TYPE trans.response.set_content_type(mime) + def handle_dataset_as_image(self, hda) -> str: + raise Exception("Unimplemented Method") -@dataproviders.decorators.has_dataproviders + +@p_dataproviders.decorators.has_dataproviders class Text(Data): edam_format = "format_2330" file_ext = 'txt' @@ -834,11 +850,10 @@ class Text(Data): with compression_utils.get_fileobj(dataset.file_name) as dataset_fh: dataset_read = dataset_fh.read(sample_size) sample_lines = dataset_read.count('\n') - est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size))) + return int(sample_lines * (float(dataset.get_size()) / float(sample_size))) except UnicodeDecodeError: log.error(f'Unable to estimate lines in file {dataset.file_name}') - est_lines = None - return est_lines + return None def count_data_lines(self, dataset): """ @@ -861,7 +876,7 @@ class Text(Data): data_lines += 1 except UnicodeDecodeError: log.error(f'Unable to count lines in file {dataset.file_name}') - data_lines = None + return None return data_lines def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None, line_wrap=True, **kwd): @@ -970,23 +985,27 @@ class Text(Data): part_file.close() # ------------- Dataproviders - @dataproviders.decorators.dataprovider_factory('line', dataproviders.line.FilteredLineDataProvider.settings) + @p_dataproviders.decorators.dataprovider_factory( + "line", p_dataproviders.line.FilteredLineDataProvider.settings + ) def line_dataprovider(self, dataset, **settings): """ Returns an iterator over the dataset's lines (that have been stripped) optionally excluding blank lines and lines that start with a comment character. """ - dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - return dataproviders.line.FilteredLineDataProvider(dataset_source, **settings) + dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) + return p_dataproviders.line.FilteredLineDataProvider(dataset_source, **settings) - @dataproviders.decorators.dataprovider_factory('regex-line', dataproviders.line.RegexLineDataProvider.settings) + @p_dataproviders.decorators.dataprovider_factory( + "regex-line", p_dataproviders.line.RegexLineDataProvider.settings + ) def regex_line_dataprovider(self, dataset, **settings): """ Returns an iterator over the dataset's lines optionally including/excluding lines that match one or more regex filters. """ - dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - return dataproviders.line.RegexLineDataProvider(dataset_source, **settings) + dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) + return p_dataproviders.line.RegexLineDataProvider(dataset_source, **settings) class Directory(Data): @@ -1084,7 +1103,7 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc count = 0 last_line_break = False - with compression_utils.get_fileobj(file_name, "U") as temp: + with compression_utils.get_fileobj(file_name) as temp: while count < LINE_COUNT: try: line = temp.readline(WIDTH) diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py index 90e6813f69d..e102ff91e54 100644 --- a/lib/galaxy/util/compression_utils.py +++ b/lib/galaxy/util/compression_utils.py @@ -5,7 +5,20 @@ import logging import os import tarfile import zipfile -from typing import Any, cast, Generator, IO, Iterable, List, Optional, Tuple, Union +from typing import ( + Any, + cast, + Generator, + IO, + Iterable, + List, + Optional, + overload, + Tuple, + Union, +) + +from typing_extensions import Literal from galaxy.util.path import safe_relpath from .checkers import ( @@ -15,7 +28,35 @@ from .checkers import ( log = logging.getLogger(__name__) -FileObjType = Union[gzip.GzipFile, bz2.BZ2File, IO[Any], io.TextIOWrapper] +FileObjTypeStr = Union[IO[str], io.TextIOWrapper] +FileObjTypeBytes = Union[gzip.GzipFile, bz2.BZ2File, IO[bytes]] +FileObjType = Union[FileObjTypeStr, FileObjTypeBytes] + + +@overload +def get_fileobj( + filename: str, mode: Literal["r"], compressed_formats: Optional[List[str]] = None +) -> FileObjTypeStr: + ... + + +@overload +def get_fileobj( + filename: str, mode: Literal["rb"], compressed_formats: Optional[List[str]] = None +) -> FileObjTypeBytes: + ... + + +@overload +def get_fileobj(filename: str) -> FileObjTypeStr: + ... + + +@overload +def get_fileobj( + filename: str, mode: str = "r", compressed_formats: Optional[List[str]] = None +) -> FileObjType: + ... def get_fileobj( @@ -33,6 +74,32 @@ def get_fileobj( return get_fileobj_raw(filename, mode, compressed_formats)[1] +@overload +def get_fileobj_raw( + filename: str, mode: Literal["r"], compressed_formats: Optional[List[str]] = None +) -> Tuple[Optional[str], FileObjTypeStr]: + ... + + +@overload +def get_fileobj_raw( + filename: str, mode: Literal["rb"], compressed_formats: Optional[List[str]] = None +) -> Tuple[Optional[str], FileObjTypeBytes]: + ... + + +@overload +def get_fileobj_raw(filename: str) -> Tuple[Optional[str], FileObjTypeStr]: + ... + + +@overload +def get_fileobj_raw( + filename: str, mode: str = "r", compressed_formats: Optional[List[str]] = None +) -> Tuple[Optional[str], FileObjType]: + ... + + def get_fileobj_raw( filename: str, mode: str = "r", compressed_formats: Optional[List[str]] = None ) -> Tuple[Optional[str], FileObjType]: diff --git a/setup.cfg b/setup.cfg index 67699e6779f..ac8a0d87bf9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -435,8 +435,6 @@ check_untyped_defs = False check_untyped_defs = False [mypy-galaxy.jobs.runners.util.cli.job.lsf] check_untyped_defs = False -[mypy-galaxy.datatypes.data] -check_untyped_defs = False [mypy-galaxy.containers.docker_swarm] check_untyped_defs = False [mypy-galaxy.authnz.managers]