Fix tell error

Fixes https://github.com/galaxyproject/galaxy/issues/13090:
```
Sun, Dec 19 2021 9:01:49 pm | Traceback (most recent call last):
Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/model/store/discover.py", line 203, in set_datasets_metadata
Sun, Dec 19 2021 9:01:49 pm | primary_data.set_meta()
Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/model/__init__.py", line 3736, in set_meta
Sun, Dec 19 2021 9:01:49 pm | return self.datatype.set_meta(self, **kwd)
Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/datatypes/interval.py", line 412, in set_meta
Sun, Dec 19 2021 9:01:49 pm | Tabular.set_meta(self, dataset, overwrite=overwrite, skip=i)
Sun, Dec 19 2021 9:01:49 pm | File "/galaxy/server/lib/galaxy/datatypes/tabular.py", line 404, in set_meta
Sun, Dec 19 2021 9:01:49 pm | if dataset_fh.tell() != dataset.get_size():
Sun, Dec 19 2021 9:01:49 pm | OSError: telling position disabled by next() call
```

Include a unit test that hits this error.
This commit is contained in:
mvdbeek
2022-01-09 18:57:10 +01:00
parent 060ab1a487
commit 85d19caa16
6 changed files with 28 additions and 7 deletions
@@ -29,14 +29,12 @@ def main():
with open(input_fname) as in_file, open(sys.argv[2], 'w') as out_file:
out_file.write('{"sections" : [')
line = in_file.readline()
while line:
for _ in iter(in_file.readline, ''):
current_line += 1
if 0 == current_line % lines_per_chunk:
chunk_end = in_file.tell()
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{sequences}"}},')
chunk_begin = chunk_end
line = in_file.readline()
chunk_end = in_file.tell()
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{current_line % lines_per_chunk / 4}"}}')
+1 -1
View File
@@ -431,7 +431,7 @@ class Fasta(Sequence):
part_file = open(part_path, 'w')
log.debug(f"Writing {input_file} part to {part_path}")
start_offset = 0
for line in f:
for line in iter(f.readline, ''):
offset = f.tell()
if not line:
break
+1 -1
View File
@@ -564,7 +564,7 @@ class FilePrefix:
def line_iterator(self):
s = self.string_io()
s_len = len(s.getvalue())
for line in s:
for line in iter(s.readline, ''):
if line.endswith("\n") or line.endswith("\r"):
yield line
elif s.tell() == s_len and not self.truncated:
+4 -2
View File
@@ -35,6 +35,8 @@ from . import dataproviders
log = logging.getLogger(__name__)
MAX_DATA_LINES = 100000
@dataproviders.decorators.has_dataproviders
class TabularData(data.Text):
@@ -270,7 +272,7 @@ class Tabular(TabularData):
def get_column_names(self, first_line=None):
return None
def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=100000, max_guess_type_data_lines=None, **kwd):
def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=MAX_DATA_LINES, max_guess_type_data_lines=None, **kwd):
"""
Tries to determine the number of columns as well as those columns that
contain numerical values in the dataset. A skip parameter is used
@@ -367,7 +369,7 @@ class Tabular(TabularData):
# NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default
with compression_utils.get_fileobj(dataset.file_name) as dataset_fh:
i = 0
for line in dataset_fh:
for line in iter(dataset_fh.readline, ''):
line = line.rstrip('\r\n')
if i == 0:
column_names = self.get_column_names(first_line=line)
+17
View File
@@ -0,0 +1,17 @@
import tempfile
from galaxy.datatypes.tabular import (
MAX_DATA_LINES,
Tabular,
)
from .util import MockDataset
def test_tabular_set_meta_large_file():
with tempfile.NamedTemporaryFile(mode='w') as test_file:
for _ in range(MAX_DATA_LINES + 1):
test_file.write("A\tB\n")
test_file.flush()
dataset = MockDataset(id=1)
dataset.file_name = test_file.name
Tabular().set_meta(dataset)
+4
View File
@@ -18,10 +18,14 @@ class MockDataset:
def __init__(self, id):
self.id = id
self.metadata = MockMetadata()
self.dataset = None
def has_data(self):
return True
def get_size(self):
return self.dataset and os.path.getsize(self.dataset.file_name)
@contextmanager
def get_dataset(filename, index_attr='bam_index', dataset_id=1, has_data=True):