treat empty files correctly

as 0 column files

- refactor type_overrules_type for simplicity
- add tests

test/unit/data/datatypes/test_tabular.py

@@ -25,10 +25,13 @@ def test_tabular_set_meta_large_file():
 
 
 def test_tabular_set_meta_empty():
+    """
+    empty file
+    """
     with tempfile.NamedTemporaryFile(mode="w") as test_file:
         test_file.flush()
         dataset = MockDataset(id=1)
-        dataset.file_name = test_file.name
+        dataset.set_file_name(test_file.name)
         Tabular().set_meta(dataset)
         # data and comment lines are not stored if more than MAX_DATA_LINES
         assert dataset.metadata.data_lines == 0
@@ -37,3 +40,75 @@ def test_tabular_set_meta_empty():
         assert dataset.metadata.columns == 0
         assert dataset.metadata.delimiter == "\t"
         assert not hasattr(dataset.metadata, "column_names")
+
+
+def test_tabular_set_meta_nearly_empty():
+    """
+    file just containing a single new line
+    - empty lines are treated as comments
+    """
+    with tempfile.NamedTemporaryFile(mode="w") as test_file:
+        test_file.write("\n")
+        test_file.flush()
+        dataset = MockDataset(id=1)
+        dataset.set_file_name(test_file.name)
+        Tabular().set_meta(dataset)
+        # data and comment lines are not stored if more than MAX_DATA_LINES
+        assert dataset.metadata.data_lines == 0
+        assert dataset.metadata.comment_lines == 1
+        assert dataset.metadata.column_types == []
+        assert dataset.metadata.columns == 0
+        assert dataset.metadata.delimiter == "\t"
+        assert not hasattr(dataset.metadata, "column_names")
+
+
+def test_tabular_column_types():
+    """
+    file just containing a single new line
+    - empty lines are treated as comments
+    """
+    with tempfile.NamedTemporaryFile(mode="w") as test_file:
+        # 1st line has special treatment which we want to ignore in this test
+        test_file.write("\t\t\t\t\n")
+        # note that the 1st column of this line will be detected as None
+        # but this is overwritten by the default column type (str) after
+        # checking all lines
+        test_file.write("\tstr\t23\t42.00\ta,b,c\n")
+        test_file.flush()
+        dataset = MockDataset(id=1)
+        dataset.set_file_name(test_file.name)
+        Tabular().set_meta(dataset)
+        # data and comment lines are not stored if more than MAX_DATA_LINES
+        assert dataset.metadata.data_lines == 2
+        assert dataset.metadata.comment_lines == 0
+        assert dataset.metadata.column_types == ["str", "str", "int", "float", "list"]
+        assert dataset.metadata.columns == 5
+        assert dataset.metadata.delimiter == "\t"
+        assert not hasattr(dataset.metadata, "column_names")
+
+
+def test_tabular_column_types_override():
+    """
+    check that guessed column types can be improved
+    by the types guessed for later lines
+    overwriting is only possible in the following order None -> int -> float -> list -> str
+
+    also check that more columns can be added by later lines
+    """
+    with tempfile.NamedTemporaryFile(mode="w") as test_file:
+        # 1st line has special treatment which we want to ignore in this test
+        test_file.write("\t\t\t\t\n")
+        # note that the first column in detected as None which can be overwritten by int
+        test_file.write("\t23\t42.00\ta,b,c\tstr\n")
+        test_file.write("23\t42.0\t23,42.0\tstr\t42\tanother column\n")
+        test_file.flush()
+        dataset = MockDataset(id=1)
+        dataset.set_file_name(test_file.name)
+        Tabular().set_meta(dataset)
+        # data and comment lines are not stored if more than MAX_DATA_LINES
+        assert dataset.metadata.data_lines == 3
+        assert dataset.metadata.comment_lines == 0
+        assert dataset.metadata.column_types == ["int", "float", "list", "str", "str", "str"]
+        assert dataset.metadata.columns == 6
+        assert dataset.metadata.delimiter == "\t"
+        assert not hasattr(dataset.metadata, "column_names")