mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Fix tell error
Fixes https://github.com/galaxyproject/galaxy/issues/13090: ``` Sun, Dec 19 2021 9:01:49 pm | Traceback (most recent call last): Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/model/store/discover.py", line 203, in set_datasets_metadata Sun, Dec 19 2021 9:01:49 pm | primary_data.set_meta() Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/model/__init__.py", line 3736, in set_meta Sun, Dec 19 2021 9:01:49 pm | return self.datatype.set_meta(self, **kwd) Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/datatypes/interval.py", line 412, in set_meta Sun, Dec 19 2021 9:01:49 pm | Tabular.set_meta(self, dataset, overwrite=overwrite, skip=i) Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/datatypes/tabular.py", line 404, in set_meta Sun, Dec 19 2021 9:01:49 pm | if dataset_fh.tell() != dataset.get_size(): Sun, Dec 19 2021 9:01:49 pm | OSError: telling position disabled by next() call ``` Include a unit test that hits this error.
This commit is contained in:
@@ -29,14 +29,12 @@ def main():
|
||||
with open(input_fname) as in_file, open(sys.argv[2], 'w') as out_file:
|
||||
out_file.write('{"sections" : [')
|
||||
|
||||
line = in_file.readline()
|
||||
while line:
|
||||
for _ in iter(in_file.readline, ''):
|
||||
current_line += 1
|
||||
if 0 == current_line % lines_per_chunk:
|
||||
chunk_end = in_file.tell()
|
||||
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{sequences}"}},')
|
||||
chunk_begin = chunk_end
|
||||
line = in_file.readline()
|
||||
|
||||
chunk_end = in_file.tell()
|
||||
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{current_line % lines_per_chunk / 4}"}}')
|
||||
|
||||
@@ -431,7 +431,7 @@ class Fasta(Sequence):
|
||||
part_file = open(part_path, 'w')
|
||||
log.debug(f"Writing {input_file} part to {part_path}")
|
||||
start_offset = 0
|
||||
for line in f:
|
||||
for line in iter(f.readline, ''):
|
||||
offset = f.tell()
|
||||
if not line:
|
||||
break
|
||||
|
||||
@@ -564,7 +564,7 @@ class FilePrefix:
|
||||
def line_iterator(self):
|
||||
s = self.string_io()
|
||||
s_len = len(s.getvalue())
|
||||
for line in s:
|
||||
for line in iter(s.readline, ''):
|
||||
if line.endswith("\n") or line.endswith("\r"):
|
||||
yield line
|
||||
elif s.tell() == s_len and not self.truncated:
|
||||
|
||||
@@ -35,6 +35,8 @@ from . import dataproviders
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_DATA_LINES = 100000
|
||||
|
||||
|
||||
@dataproviders.decorators.has_dataproviders
|
||||
class TabularData(data.Text):
|
||||
@@ -270,7 +272,7 @@ class Tabular(TabularData):
|
||||
def get_column_names(self, first_line=None):
|
||||
return None
|
||||
|
||||
def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=100000, max_guess_type_data_lines=None, **kwd):
|
||||
def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=MAX_DATA_LINES, max_guess_type_data_lines=None, **kwd):
|
||||
"""
|
||||
Tries to determine the number of columns as well as those columns that
|
||||
contain numerical values in the dataset. A skip parameter is used
|
||||
@@ -367,7 +369,7 @@ class Tabular(TabularData):
|
||||
# NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default
|
||||
with compression_utils.get_fileobj(dataset.file_name) as dataset_fh:
|
||||
i = 0
|
||||
for line in dataset_fh:
|
||||
for line in iter(dataset_fh.readline, ''):
|
||||
line = line.rstrip('\r\n')
|
||||
if i == 0:
|
||||
column_names = self.get_column_names(first_line=line)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import tempfile
|
||||
|
||||
from galaxy.datatypes.tabular import (
|
||||
MAX_DATA_LINES,
|
||||
Tabular,
|
||||
)
|
||||
from .util import MockDataset
|
||||
|
||||
|
||||
def test_tabular_set_meta_large_file():
|
||||
with tempfile.NamedTemporaryFile(mode='w') as test_file:
|
||||
for _ in range(MAX_DATA_LINES + 1):
|
||||
test_file.write("A\tB\n")
|
||||
test_file.flush()
|
||||
dataset = MockDataset(id=1)
|
||||
dataset.file_name = test_file.name
|
||||
Tabular().set_meta(dataset)
|
||||
@@ -18,10 +18,14 @@ class MockDataset:
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
self.metadata = MockMetadata()
|
||||
self.dataset = None
|
||||
|
||||
def has_data(self):
|
||||
return True
|
||||
|
||||
def get_size(self):
|
||||
return self.dataset and os.path.getsize(self.dataset.file_name)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_dataset(filename, index_attr='bam_index', dataset_id=1, has_data=True):
|
||||
|
||||
Reference in New Issue
Block a user