From 9f04af05cd8390316db85802195639939be5ae29 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 15 May 2022 10:49:45 +0200 Subject: [PATCH 1/5] extend existing unit test for Tabular.set_meta to actually check set metadata values --- test/unit/data/datatypes/test_tabular.py | 7 +++++++ test/unit/data/datatypes/util.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/unit/data/datatypes/test_tabular.py b/test/unit/data/datatypes/test_tabular.py index d860ec9f456..6298e3e7e07 100644 --- a/test/unit/data/datatypes/test_tabular.py +++ b/test/unit/data/datatypes/test_tabular.py @@ -15,3 +15,10 @@ def test_tabular_set_meta_large_file(): dataset = MockDataset(id=1) dataset.set_file_name(test_file.name) Tabular().set_meta(dataset) # type: ignore [arg-type] + # data and comment lines are not stored if more than MAX_DATA_LINES + assert dataset.metadata.data_lines is None + assert dataset.metadata.comment_lines is None + assert dataset.metadata.column_types == ["str", "str"] + assert dataset.metadata.columns == 2 + assert dataset.metadata.delimiter == "\t" + assert not hasattr(dataset.metadata, "column_names") diff --git a/test/unit/data/datatypes/util.py b/test/unit/data/datatypes/util.py index ff0de253899..01b496435b5 100644 --- a/test/unit/data/datatypes/util.py +++ b/test/unit/data/datatypes/util.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from typing import Optional from galaxy.datatypes.sniff import get_test_fname +from galaxy.util.bunch import Bunch from galaxy.util.hash_util import md5_hash_file @@ -20,7 +21,7 @@ class MockDatasetDataset: self.file_name_ = file_name -class MockMetadata: +class MockMetadata(Bunch): file_name_: Optional[str] = None def get_file_name(self, sync_cache=True): From b7ca5258a3b78ad59cd6b07e19c19191df4be492 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 15 May 2022 13:26:36 +0200 Subject: [PATCH 2/5] add unit test for Tabular.set_meta for empty files --- lib/galaxy/datatypes/tabular.py | 3 ++- test/unit/data/datatypes/test_tabular.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 87d709e7599..6e52675b396 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -556,13 +556,14 @@ class Tabular(TabularData): comment_lines = None # type: ignore [assignment] break i += 1 - + print(column_types) # we error on the larger number of columns # first we pad our column_types by using data from first line if len(first_line_column_types) > len(column_types): for column_type in first_line_column_types[len(column_types) :]: column_types.append(column_type) # Now we fill any unknown (None) column_types with data from first line + print(column_types) for i in range(len(column_types)): if column_types[i] is None: if len(first_line_column_types) <= i or first_line_column_types[i] is None: diff --git a/test/unit/data/datatypes/test_tabular.py b/test/unit/data/datatypes/test_tabular.py index 6298e3e7e07..002e8509aca 100644 --- a/test/unit/data/datatypes/test_tabular.py +++ b/test/unit/data/datatypes/test_tabular.py @@ -22,3 +22,18 @@ def test_tabular_set_meta_large_file(): assert dataset.metadata.columns == 2 assert dataset.metadata.delimiter == "\t" assert not hasattr(dataset.metadata, "column_names") + + +def test_tabular_set_meta_empty(): + with tempfile.NamedTemporaryFile(mode="w") as test_file: + test_file.flush() + dataset = MockDataset(id=1) + dataset.file_name = test_file.name + Tabular().set_meta(dataset) + # data and comment lines are not stored if more than MAX_DATA_LINES + assert dataset.metadata.data_lines == 0 + assert dataset.metadata.comment_lines == 0 + assert dataset.metadata.column_types == [] + assert dataset.metadata.columns == 0 + assert dataset.metadata.delimiter == "\t" + assert not hasattr(dataset.metadata, "column_names") From d90a1013311d79b647a95d4caaf13e15544d472e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 15 May 2022 14:23:17 +0200 Subject: [PATCH 3/5] treat empty files correctly as 0 column files - refactor type_overrules_type for simplicity - add tests --- lib/galaxy/datatypes/tabular.py | 17 +++--- test/unit/data/datatypes/test_tabular.py | 77 +++++++++++++++++++++++- 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 6e52675b396..0e3ed958951 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -450,18 +450,18 @@ class Tabular(TabularData): column_type_compare_order = list(column_type_set_order) # Order to compare column types column_type_compare_order.reverse() - def type_overrules_type(column_type1, column_type2): - if column_type1 is None or column_type1 == column_type2: + def type_overrules_type(new_column_type, old_column_type): + if new_column_type is None or new_column_type == old_column_type: return False - if column_type2 is None: + if old_column_type is None: return True for column_type in column_type_compare_order: - if column_type1 == column_type: + if new_column_type == column_type: return True - if column_type2 == column_type: + if old_column_type == column_type: return False # neither column type was found in our ordered list, this cannot happen - raise ValueError(f"Tried to compare unknown column types: {column_type1} and {column_type2}") + raise ValueError(f"Tried to compare unknown column types: {new_column_type} and {old_column_type}") def is_int(column_text): # Don't allow underscores in numeric literals (PEP 515) @@ -508,7 +508,7 @@ class Tabular(TabularData): comment_lines = 0 column_names = None column_types: List = [] - first_line_column_types = [default_column_type] # default value is one column of type str + first_line_column_types = [] if dataset.has_data(): # NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default with compression_utils.get_fileobj(dataset.get_file_name()) as dataset_fh: @@ -556,14 +556,13 @@ class Tabular(TabularData): comment_lines = None # type: ignore [assignment] break i += 1 - print(column_types) + # we error on the larger number of columns # first we pad our column_types by using data from first line if len(first_line_column_types) > len(column_types): for column_type in first_line_column_types[len(column_types) :]: column_types.append(column_type) # Now we fill any unknown (None) column_types with data from first line - print(column_types) for i in range(len(column_types)): if column_types[i] is None: if len(first_line_column_types) <= i or first_line_column_types[i] is None: diff --git a/test/unit/data/datatypes/test_tabular.py b/test/unit/data/datatypes/test_tabular.py index 002e8509aca..0e3fe65b112 100644 --- a/test/unit/data/datatypes/test_tabular.py +++ b/test/unit/data/datatypes/test_tabular.py @@ -25,10 +25,13 @@ def test_tabular_set_meta_large_file(): def test_tabular_set_meta_empty(): + """ + empty file + """ with tempfile.NamedTemporaryFile(mode="w") as test_file: test_file.flush() dataset = MockDataset(id=1) - dataset.file_name = test_file.name + dataset.set_file_name(test_file.name) Tabular().set_meta(dataset) # data and comment lines are not stored if more than MAX_DATA_LINES assert dataset.metadata.data_lines == 0 @@ -37,3 +40,75 @@ def test_tabular_set_meta_empty(): assert dataset.metadata.columns == 0 assert dataset.metadata.delimiter == "\t" assert not hasattr(dataset.metadata, "column_names") + + +def test_tabular_set_meta_nearly_empty(): + """ + file just containing a single new line + - empty lines are treated as comments + """ + with tempfile.NamedTemporaryFile(mode="w") as test_file: + test_file.write("\n") + test_file.flush() + dataset = MockDataset(id=1) + dataset.set_file_name(test_file.name) + Tabular().set_meta(dataset) + # data and comment lines are not stored if more than MAX_DATA_LINES + assert dataset.metadata.data_lines == 0 + assert dataset.metadata.comment_lines == 1 + assert dataset.metadata.column_types == [] + assert dataset.metadata.columns == 0 + assert dataset.metadata.delimiter == "\t" + assert not hasattr(dataset.metadata, "column_names") + + +def test_tabular_column_types(): + """ + file just containing a single new line + - empty lines are treated as comments + """ + with tempfile.NamedTemporaryFile(mode="w") as test_file: + # 1st line has special treatment which we want to ignore in this test + test_file.write("\t\t\t\t\n") + # note that the 1st column of this line will be detected as None + # but this is overwritten by the default column type (str) after + # checking all lines + test_file.write("\tstr\t23\t42.00\ta,b,c\n") + test_file.flush() + dataset = MockDataset(id=1) + dataset.set_file_name(test_file.name) + Tabular().set_meta(dataset) + # data and comment lines are not stored if more than MAX_DATA_LINES + assert dataset.metadata.data_lines == 2 + assert dataset.metadata.comment_lines == 0 + assert dataset.metadata.column_types == ["str", "str", "int", "float", "list"] + assert dataset.metadata.columns == 5 + assert dataset.metadata.delimiter == "\t" + assert not hasattr(dataset.metadata, "column_names") + + +def test_tabular_column_types_override(): + """ + check that guessed column types can be improved + by the types guessed for later lines + overwriting is only possible in the following order None -> int -> float -> list -> str + + also check that more columns can be added by later lines + """ + with tempfile.NamedTemporaryFile(mode="w") as test_file: + # 1st line has special treatment which we want to ignore in this test + test_file.write("\t\t\t\t\n") + # note that the first column in detected as None which can be overwritten by int + test_file.write("\t23\t42.00\ta,b,c\tstr\n") + test_file.write("23\t42.0\t23,42.0\tstr\t42\tanother column\n") + test_file.flush() + dataset = MockDataset(id=1) + dataset.set_file_name(test_file.name) + Tabular().set_meta(dataset) + # data and comment lines are not stored if more than MAX_DATA_LINES + assert dataset.metadata.data_lines == 3 + assert dataset.metadata.comment_lines == 0 + assert dataset.metadata.column_types == ["int", "float", "list", "str", "str", "str"] + assert dataset.metadata.columns == 6 + assert dataset.metadata.delimiter == "\t" + assert not hasattr(dataset.metadata, "column_names") From f19818435007dbc20ddb64016349df71f7624eea Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 19 Feb 2024 16:36:00 +0100 Subject: [PATCH 4/5] add type ignores for mock --- test/unit/data/datatypes/test_tabular.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit/data/datatypes/test_tabular.py b/test/unit/data/datatypes/test_tabular.py index 0e3fe65b112..8a731266974 100644 --- a/test/unit/data/datatypes/test_tabular.py +++ b/test/unit/data/datatypes/test_tabular.py @@ -32,7 +32,7 @@ def test_tabular_set_meta_empty(): test_file.flush() dataset = MockDataset(id=1) dataset.set_file_name(test_file.name) - Tabular().set_meta(dataset) + Tabular().set_meta(dataset) # type: ignore [arg-type] # data and comment lines are not stored if more than MAX_DATA_LINES assert dataset.metadata.data_lines == 0 assert dataset.metadata.comment_lines == 0 @@ -52,7 +52,7 @@ def test_tabular_set_meta_nearly_empty(): test_file.flush() dataset = MockDataset(id=1) dataset.set_file_name(test_file.name) - Tabular().set_meta(dataset) + Tabular().set_meta(dataset) # type: ignore [arg-type] # data and comment lines are not stored if more than MAX_DATA_LINES assert dataset.metadata.data_lines == 0 assert dataset.metadata.comment_lines == 1 @@ -77,7 +77,7 @@ def test_tabular_column_types(): test_file.flush() dataset = MockDataset(id=1) dataset.set_file_name(test_file.name) - Tabular().set_meta(dataset) + Tabular().set_meta(dataset) # type: ignore [arg-type] # data and comment lines are not stored if more than MAX_DATA_LINES assert dataset.metadata.data_lines == 2 assert dataset.metadata.comment_lines == 0 @@ -104,7 +104,7 @@ def test_tabular_column_types_override(): test_file.flush() dataset = MockDataset(id=1) dataset.set_file_name(test_file.name) - Tabular().set_meta(dataset) + Tabular().set_meta(dataset) # type: ignore [arg-type] # data and comment lines are not stored if more than MAX_DATA_LINES assert dataset.metadata.data_lines == 3 assert dataset.metadata.comment_lines == 0 From 25d95278cb4e435bffbdbff83c74987c3b0e51cf Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 1 Mar 2024 13:39:51 -0500 Subject: [PATCH 5/5] Make comment more detailed --- test/unit/data/datatypes/test_tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/data/datatypes/test_tabular.py b/test/unit/data/datatypes/test_tabular.py index 8a731266974..c98fc1c6f7e 100644 --- a/test/unit/data/datatypes/test_tabular.py +++ b/test/unit/data/datatypes/test_tabular.py @@ -64,7 +64,7 @@ def test_tabular_set_meta_nearly_empty(): def test_tabular_column_types(): """ - file just containing a single new line + file containing a single containing only tab characters terminated with a new line character - empty lines are treated as comments """ with tempfile.NamedTemporaryFile(mode="w") as test_file: