diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py index dfc4d4ffa66..cb0fb34b0e9 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py +++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py @@ -29,14 +29,12 @@ def main(): with open(input_fname) as in_file, open(sys.argv[2], 'w') as out_file: out_file.write('{"sections" : [') - line = in_file.readline() - while line: + for _ in iter(in_file.readline, ''): current_line += 1 if 0 == current_line % lines_per_chunk: chunk_end = in_file.tell() out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{sequences}"}},') chunk_begin = chunk_end - line = in_file.readline() chunk_end = in_file.tell() out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{current_line % lines_per_chunk / 4}"}}') diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 0a410c01b70..875c7c5730e 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -431,7 +431,7 @@ class Fasta(Sequence): part_file = open(part_path, 'w') log.debug(f"Writing {input_file} part to {part_path}") start_offset = 0 - for line in f: + for line in iter(f.readline, ''): offset = f.tell() if not line: break diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 7f5bee07975..40341e0bc6b 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -564,7 +564,7 @@ class FilePrefix: def line_iterator(self): s = self.string_io() s_len = len(s.getvalue()) - for line in s: + for line in iter(s.readline, ''): if line.endswith("\n") or line.endswith("\r"): yield line elif s.tell() == s_len and not self.truncated: diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index d7f6f070abe..cd3493c77ac 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -35,6 +35,8 @@ from . import dataproviders log = logging.getLogger(__name__) +MAX_DATA_LINES = 100000 + @dataproviders.decorators.has_dataproviders class TabularData(data.Text): @@ -270,7 +272,7 @@ class Tabular(TabularData): def get_column_names(self, first_line=None): return None - def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=100000, max_guess_type_data_lines=None, **kwd): + def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=MAX_DATA_LINES, max_guess_type_data_lines=None, **kwd): """ Tries to determine the number of columns as well as those columns that contain numerical values in the dataset. A skip parameter is used @@ -367,7 +369,7 @@ class Tabular(TabularData): # NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default with compression_utils.get_fileobj(dataset.file_name) as dataset_fh: i = 0 - for line in dataset_fh: + for line in iter(dataset_fh.readline, ''): line = line.rstrip('\r\n') if i == 0: column_names = self.get_column_names(first_line=line) diff --git a/test/unit/data/datatypes/test_tabular.py b/test/unit/data/datatypes/test_tabular.py new file mode 100644 index 00000000000..3875e94f674 --- /dev/null +++ b/test/unit/data/datatypes/test_tabular.py @@ -0,0 +1,17 @@ +import tempfile + +from galaxy.datatypes.tabular import ( + MAX_DATA_LINES, + Tabular, +) +from .util import MockDataset + + +def test_tabular_set_meta_large_file(): + with tempfile.NamedTemporaryFile(mode='w') as test_file: + for _ in range(MAX_DATA_LINES + 1): + test_file.write("A\tB\n") + test_file.flush() + dataset = MockDataset(id=1) + dataset.file_name = test_file.name + Tabular().set_meta(dataset) diff --git a/test/unit/data/datatypes/util.py b/test/unit/data/datatypes/util.py index 571264f2f10..d5d8b86a84a 100644 --- a/test/unit/data/datatypes/util.py +++ b/test/unit/data/datatypes/util.py @@ -18,10 +18,14 @@ class MockDataset: def __init__(self, id): self.id = id self.metadata = MockMetadata() + self.dataset = None def has_data(self): return True + def get_size(self): + return self.dataset and os.path.getsize(self.dataset.file_name) + @contextmanager def get_dataset(filename, index_attr='bam_index', dataset_id=1, has_data=True):