Set webdav plugin to use temp files by default

it avoids the issue with loading the entire files into memory.
This commit is contained in:
davelopez
2024-01-30 13:43:09 +01:00
parent ddf992cbfd
commit 5153394f15
+6
View File
@@ -3,6 +3,7 @@ try:
except ImportError:
WebDAVFS = None
import tempfile
from typing import (
Optional,
Union,
@@ -22,6 +23,11 @@ class WebDavFilesSource(PyFilesystem2FilesSource):
def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
props = self._serialization_props(user_context)
use_temp_files = props.pop("use_temp_files", None)
if use_temp_files is None:
# Default to True to avoid memory issues with large files.
props["use_temp_files"] = True
props["temp_path"] = props.get("temp_path", tempfile.gettempdir())
extra_props: Union[FilesSourceProperties, dict] = opts.extra_props or {} if opts else {}
handle = WebDAVFS(**{**props, **extra_props})
return handle