opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Type fixes for genome data providers.

Browse Source
This commit is contained in:
John Chilton
2021-10-26 10:27:58 -04:00
parent b2728d4ab3
commit ad7933d926
3 changed files with 39 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/visualization/data_providers/basic.py
+4 -1
View File
@@ -1,7 +1,9 @@
import sys
from json import loads
from typing import Iterator
from galaxy.datatypes.tabular import Tabular
from galaxy.model import DatasetInstance
class BaseDataProvider:
@@ -12,6 +14,7 @@ class BaseDataProvider:
- write subsets of data to new datasets
"""
original_dataset: DatasetInstance
def __init__(self, converted_dataset=None, original_dataset=None, dependencies=None,
error_max_vals="Only the first %i values are returned."):
@@ -28,7 +31,7 @@ class BaseDataProvider:
"""
raise Exception("Unimplemented Function")
def get_iterator(self, **kwargs):
def get_iterator(self, data_file, chrom, start, end, **kwargs) -> Iterator[str]:
"""
Returns an iterator that provides data in the region chrom:start-end
"""
lib/galaxy/visualization/data_providers/genome.py
+35 -25
View File
@@ -10,7 +10,7 @@ import re
import sys
from contextlib import contextmanager
from json import loads
from typing import Dict, Union
from typing import Any, Dict, IO, Iterator, List, Optional, Tuple, Union
import pysam
from bx.bbi.bigbed_file import BigBedFile
@@ -19,6 +19,7 @@ from bx.interval_index_file import Indexes
from galaxy.datatypes.interval import Bed, Gff, Gtf
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes
from galaxy.model import DatasetInstance
from galaxy.visualization.data_providers.basic import BaseDataProvider
from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and_cigar
@@ -30,6 +31,8 @@ from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and
# Can be be removed once https://github.com/pysam-developers/pysam/issues/939 is resolved.
pysam.set_verbosity(0)
PAYLOAD_LIST_TYPE = List[Optional[Union[str, int, float, List[Tuple[int, int]]]]]
def float_nan(n):
'''
@@ -89,9 +92,9 @@ class FeatureLocationIndexDataProvider(BaseDataProvider):
# Find query in file using binary search.
low = 0
high = file_len / line_len
high = int(file_len / line_len)
while low < high:
mid = (low + high) // 2
mid: int = (low + high) // 2
position = mid * line_len
textloc_file.seek(position)
@@ -171,7 +174,7 @@ class GenomeDataProvider(BaseDataProvider):
"""
raise Exception("Unimplemented Function")
def get_iterator(self, data_file, chrom, start, end, **kwargs):
def get_iterator(self, data_file, chrom, start, end, **kwargs) -> Iterator[str]:
"""
Returns an iterator that provides data in the region chrom:start-end
"""
@@ -272,6 +275,8 @@ class GenomeDataProvider(BaseDataProvider):
class FilterableMixin:
original_dataset: DatasetInstance
def get_filters(self):
""" Returns a dataset's filters. """
# Get filters.
@@ -329,7 +334,7 @@ class TabixDataProvider(GenomeDataProvider, FilterableMixin):
with pysam.TabixFile(self.dependencies['bgzip'].file_name, index=index_path) as f:
yield f
def get_iterator(self, data_file, chrom, start, end, **kwargs):
def get_iterator(self, data_file, chrom, start, end, **kwargs) -> Iterator[str]:
# chrom must be a string, start/end integers.
# in previous versions of pysam, unicode was accepted for chrom, but not in 8.4
chrom = str(chrom)
@@ -338,7 +343,7 @@ class TabixDataProvider(GenomeDataProvider, FilterableMixin):
if end >= (2 << 29):
end = (2 << 29 - 1) # Tabix-enforced maximum
# Get iterator using either naming scheme.
iterator = iter([])
iterator: Iterator[str] = iter([])
if chrom in data_file.contigs:
iterator = data_file.fetch(reference=chrom, start=start, end=end)
else:
@@ -406,7 +411,7 @@ class IntervalDataProvider(GenomeDataProvider):
feature = line.split()
length = len(feature)
# Unique id is just a hash of the line
payload = [hash(line), int(feature[start_col]), int(feature[end_col])]
payload: PAYLOAD_LIST_TYPE = [hash(line), int(feature[start_col]), int(feature[end_col])]
if no_detail:
rval.append(payload)
@@ -484,7 +489,7 @@ class BedDataProvider(GenomeDataProvider):
feature = line.split()
length = len(feature)
# Unique id is just a hash of the line
payload = [hash(line), int(feature[1]), int(feature[2])]
payload: PAYLOAD_LIST_TYPE = [hash(line), int(feature[1]), int(feature[2])]
if no_detail:
rval.append(payload)
@@ -549,7 +554,7 @@ class RawBedDataProvider(BedDataProvider):
for large datasets.
"""
def get_iterator(self, data_file, chrom=None, start=None, end=None, **kwargs):
def get_iterator(self, data_file, chrom, start, end, **kwargs):
# Read first line in order to match chrom naming format.
line = data_file.readline()
dataset_chrom = line.split()[0]
@@ -677,7 +682,7 @@ class VcfDataProvider(GenomeDataProvider):
if samples_data:
# Process and pack samples' genotype and count alleles across samples.
alleles_seen = {}
alleles_seen: Dict[int, bool] = {}
has_alleles = False
for sample in samples_data:
@@ -685,10 +690,10 @@ class VcfDataProvider(GenomeDataProvider):
genotype = sample.split(':')[0]
has_alleles = False
alleles_seen.clear()
for allele in genotype_re.split(genotype):
for allele_str in genotype_re.split(genotype):
try:
# This may throw a ValueError if allele is missing.
allele = int(allele)
allele = int(allele_str)
# Only count allele if it hasn't been seen yet.
if allele != 0 and allele not in alleles_seen:
@@ -856,7 +861,7 @@ class BamDataProvider(GenomeDataProvider, FilterableMixin):
index_filename=self.converted_dataset.file_name) as f:
yield f
def get_iterator(self, data_file, chrom, start, end, **kwargs):
def get_iterator(self, data_file, chrom, start, end, **kwargs) -> Iterator[str]:
"""
Returns an iterator that provides data in the region chrom:start-end
"""
@@ -873,7 +878,7 @@ class BamDataProvider(GenomeDataProvider, FilterableMixin):
try:
data = data_file.fetch(start=start, end=end, reference=chrom)
except ValueError:
return None
return iter([])
return data
def process_data(self, iterator, start_val=0, max_vals=None, ref_seq=None,
@@ -966,7 +971,7 @@ class BamDataProvider(GenomeDataProvider, FilterableMixin):
# Encode reads as list of lists.
#
results = []
paired_pending = {}
paired_pending: Dict[str, Dict[str, Any]] = {}
unmapped = 0
message = None
count = 0
@@ -1110,6 +1115,9 @@ class BBIDataProvider(GenomeDataProvider):
dataset_type = 'bigwig'
def _get_dataset(self) -> Tuple[IO[bytes], Union[BigBedFile, BigWigFile]]:
...
def valid_chroms(self):
# No way to return this info as of now
return None
@@ -1146,7 +1154,7 @@ class BBIDataProvider(GenomeDataProvider):
min_val = 0
max_val = 0
mean = 0
sd = 0
sd = 0.0
if summary is not None:
# Does the summary contain any defined values?
valid_count = summary.valid_count[0]
@@ -1288,7 +1296,7 @@ class IntervalIndexDataProvider(GenomeDataProvider, FilterableMixin):
i = Indexes(self.converted_dataset.file_name)
yield i
def get_iterator(self, data_file, chrom, start, end, **kwargs):
def get_iterator(self, data_file, chrom, start, end, **kwargs) -> Iterator[str]:
"""
Returns an iterator for data in data_file in chrom:start-end
"""
@@ -1406,16 +1414,18 @@ class GtfTabixDataProvider(TabixDataProvider):
# TODO: extend this code or use code in gff_util to process GFF/3 as well
# and then create a generic GFFDataProvider that can be used with both
# raw and tabix datasets.
features = {}
features: Dict[str, List[GFFInterval]] = {}
for line in iterator:
line_attrs = parse_gff_attributes(line.split('\t')[8])
transcript_id = line_attrs['transcript_id']
feature_list: List[GFFInterval]
if transcript_id in features:
feature = features[transcript_id]
feature_list = features[transcript_id]
else:
feature = []
features[transcript_id] = feature
feature.append(GFFInterval(None, line.split('\t')))
feature_list = []
features[transcript_id] = feature_list
feature_list.append(GFFInterval(None, line.split('\t')))
# Process data.
filter_cols = loads(kwargs.get("filter_cols", "[]"))
@@ -1480,7 +1490,7 @@ class ENCODEPeakDataProvider(GenomeDataProvider):
feature = line.split()
# Feature initialization.
payload = [
payload: PAYLOAD_LIST_TYPE = [
# GUID is just a hash of the line
hash(line),
# Add start, end.
@@ -1597,7 +1607,7 @@ class ChromatinInteractionsDataProvider(GenomeDataProvider):
class ChromatinInteractionsTabixDataProvider(TabixDataProvider, ChromatinInteractionsDataProvider):
def get_iterator(self, data_file, chrom, start=0, end=sys.maxsize, interchromosomal=False, **kwargs):
def get_iterator(self, data_file, chrom, start=0, end=sys.maxsize, interchromosomal=False, **kwargs) -> Iterator[str]:
"""
"""
# Modify start as needed to get earlier interactions with start region.
@@ -1625,7 +1635,7 @@ class ChromatinInteractionsTabixDataProvider(TabixDataProvider, ChromatinInterac
#
def package_gff_feature(feature, no_detail=False, filter_cols=None):
def package_gff_feature(feature, no_detail=False, filter_cols=None) -> PAYLOAD_LIST_TYPE:
""" Package a GFF feature in an array for data providers. """
filter_cols = filter_cols or []
feature = convert_gff_coords_to_bed(feature)
setup.cfg
-2
View File
@@ -503,8 +503,6 @@ check_untyped_defs = False
check_untyped_defs = False
[mypy-tool_shed.utility_containers]
check_untyped_defs = False
[mypy-galaxy.visualization.data_providers.genome]
check_untyped_defs = False
[mypy-galaxy.datatypes.registry]
check_untyped_defs = False
[mypy-galaxy.tools.data_fetch]