diff --git a/lib/galaxy/files/sources/__init__.py b/lib/galaxy/files/sources/__init__.py index 43dfa26b0d9..9a9639c70e1 100644 --- a/lib/galaxy/files/sources/__init__.py +++ b/lib/galaxy/files/sources/__init__.py @@ -185,15 +185,11 @@ class BaseFilesSource(FilesSource): else: return ctime.strftime("%m/%d/%Y %I:%M:%S %p") + @abc.abstractmethod def _serialization_props(self, user_context=None): """Serialize properties needed to recover plugin configuration. - Used in to_dict method if for_serialization is True. """ - effective_props = {} - for key, val in self._props.items(): - effective_props[key] = self._evaluate_prop(val, user_context=user_context) - return effective_props def list(self, path="/", recursive=False, user_context=None, **kwargs): self._check_user_access(user_context) diff --git a/lib/galaxy/files/sources/_pyfilesystem2.py b/lib/galaxy/files/sources/_pyfilesystem2.py index 4e87dd02e41..de7e899e1d2 100644 --- a/lib/galaxy/files/sources/_pyfilesystem2.py +++ b/lib/galaxy/files/sources/_pyfilesystem2.py @@ -79,3 +79,9 @@ class PyFilesystem2FilesSource(BaseFilesSource): "uri": uri, "path": path, } + + def _serialization_props(self, user_context=None): + effective_props = {} + for key, val in self._props.items(): + effective_props[key] = self._evaluate_prop(val, user_context=user_context) + return effective_props diff --git a/lib/galaxy/files/sources/anvil.py b/lib/galaxy/files/sources/anvil.py index b1491c00f3c..e715366f322 100644 --- a/lib/galaxy/files/sources/anvil.py +++ b/lib/galaxy/files/sources/anvil.py @@ -10,7 +10,7 @@ class AnVILFilesSource(PyFilesystem2FilesSource): required_module = AnVILFS required_package = "fs.anvilfs" - def _open_fs(self, user_context, extra_props=None): + def _open_fs(self, user_context, kwargs=None): props = self._serialization_props(user_context) extra_props = kwargs.get("extra_props") or {} handle = AnVILFS(**{**props, **extra_props}) diff --git a/lib/galaxy/files/sources/base64.py b/lib/galaxy/files/sources/base64.py index 0f853b81916..90460e907e2 100644 --- a/lib/galaxy/files/sources/base64.py +++ b/lib/galaxy/files/sources/base64.py @@ -34,5 +34,11 @@ class Base64FilesSource(BaseFilesSource): else: return 0 + def _serialization_props(self, user_context=None): + effective_props = {} + for key, val in self._props.items(): + effective_props[key] = self._evaluate_prop(val, user_context=user_context) + return effective_props + __all__ = ("Base64FilesSource",) diff --git a/lib/galaxy/files/sources/drs.py b/lib/galaxy/files/sources/drs.py index aa9a848f43f..bed5e59d6cc 100644 --- a/lib/galaxy/files/sources/drs.py +++ b/lib/galaxy/files/sources/drs.py @@ -35,5 +35,11 @@ class DRSFilesSource(BaseFilesSource): else: return 0 + def _serialization_props(self, user_context=None): + effective_props = {} + for key, val in self._props.items(): + effective_props[key] = self._evaluate_prop(val, user_context=user_context) + return effective_props + __all__ = ("DRSFilesSource",) diff --git a/lib/galaxy/files/sources/http.py b/lib/galaxy/files/sources/http.py index 4c232c4ef15..dc998f46aca 100644 --- a/lib/galaxy/files/sources/http.py +++ b/lib/galaxy/files/sources/http.py @@ -44,7 +44,9 @@ class HTTPFilesSource(BaseFilesSource): raise NotImplementedError() def _serialization_props(self, user_context=None): - effective_props = super()._serialization_props(user_context=user_context) + effective_props = {} + for key, val in self._props.items(): + effective_props[key] = self._evaluate_prop(val, user_context=user_context) effective_props["url_regex"] = self._url_regex_str return effective_props diff --git a/lib/galaxy/files/sources/s3fs.py b/lib/galaxy/files/sources/s3fs.py index db6565a327a..8051db5f11c 100644 --- a/lib/galaxy/files/sources/s3fs.py +++ b/lib/galaxy/files/sources/s3fs.py @@ -86,7 +86,9 @@ class S3FsFilesSource(BaseFilesSource): } def _serialization_props(self, user_context=None): - effective_props = super()._serialization_props(user_context=user_context) + effective_props = {} + for key, val in self._props.items(): + effective_props[key] = self._evaluate_prop(val, user_context=user_context) effective_props["bucket"] = self._bucket return effective_props diff --git a/lib/galaxy/files/uris.py b/lib/galaxy/files/uris.py index abfbad0bfef..8001235f5b3 100644 --- a/lib/galaxy/files/uris.py +++ b/lib/galaxy/files/uris.py @@ -46,7 +46,7 @@ def stream_url_to_file( target_path: Optional[str] = None, **kwargs: dict ) -> str: - if not file_sources: + if file_sources is None: file_sources = ConfiguredFileSources.from_dict(None, load_stock_plugins=True) file_source, rel_path = file_sources.get_file_source_path(url) if file_source: diff --git a/test/unit/files/test_drs.py b/test/unit/files/test_drs.py index 87189c3c689..7846f3af6ba 100644 --- a/test/unit/files/test_drs.py +++ b/test/unit/files/test_drs.py @@ -2,6 +2,7 @@ import io import json import os import urllib +from typing import Any from unittest import mock import responses @@ -43,7 +44,7 @@ def test_file_source_drs_http(): def check_specific_header(request, **kwargs): assert request.full_url == "https://my.respository.org/myfile.txt" assert request.headers["Authorization"] == "Basic Z2E0Z2g6ZHJz" - response = io.StringIO("hello drs world") + response: Any = io.StringIO("hello drs world") response.headers = {} return response diff --git a/test/unit/files/test_ftp.py b/test/unit/files/test_ftp.py index 7d8b5cd6675..55a519b5540 100644 --- a/test/unit/files/test_ftp.py +++ b/test/unit/files/test_ftp.py @@ -1,9 +1,6 @@ -import io import os -import urllib -from unittest import mock -from ._util import configured_file_sources, user_context_fixture, assert_realizes_as, assert_realizes_contains +from ._util import configured_file_sources, user_context_fixture, assert_realizes_contains SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "ftp_file_sources_conf.yml") diff --git a/test/unit/files/test_http.py b/test/unit/files/test_http.py index 7b56b9da561..688ea48fe4f 100644 --- a/test/unit/files/test_http.py +++ b/test/unit/files/test_http.py @@ -1,6 +1,7 @@ import io import os import urllib +from typing import Any from unittest import mock from ._util import configured_file_sources, user_context_fixture, assert_realizes_as, assert_realizes_contains @@ -13,7 +14,7 @@ def test_file_source_http_specific(): def check_specific_header(request, **kwargs): assert request.headers["Authorization"] == "Bearer IBearTokens" - response = io.StringIO('hello specific world') + response: Any = io.StringIO('hello specific world') response.headers = {} return response @@ -33,7 +34,7 @@ def test_file_source_another_http_specific(): def check_another_header(request, **kwargs): assert request.headers["Another_header"] == "found" - response = io.StringIO('hello another world') + response: Any = io.StringIO('hello another world') response.headers = {} return response @@ -53,7 +54,7 @@ def test_file_source_http_generic(): def check_generic_headers(request, **kwargs): assert not request.headers - response = io.StringIO('hello generic world') + response: Any = io.StringIO('hello generic world') response.headers = {} return response @@ -73,7 +74,7 @@ def test_file_source_ftp_url(): def check_generic_headers(request, **kwargs): assert not request.headers - response = io.StringIO('This is ftp.gnu.org, the FTP server of the the GNU project.') + response: Any = io.StringIO('This is ftp.gnu.org, the FTP server of the the GNU project.') response.headers = {} return response diff --git a/test/unit/files/test_s3.py b/test/unit/files/test_s3.py index 08f4676beb4..8fb0b55f28c 100644 --- a/test/unit/files/test_s3.py +++ b/test/unit/files/test_s3.py @@ -21,6 +21,7 @@ def test_file_source(): contains=True, ) + def test_file_source_generic(): file_url = "s3://ga4gh-demo-data/phenopackets/Cao-2018-TGFBR2-Patient_4.json" user_context = user_context_fixture() @@ -34,6 +35,7 @@ def test_file_source_generic(): file_sources, file_url, "PMID:30101859-Cao-2018-TGFBR2-Patient_4", user_context=user_context ) + def test_file_source_specific(): file_url = "s3://genomeark/data_use_policies.txt" user_context = user_context_fixture() @@ -43,6 +45,4 @@ def test_file_source_specific(): assert file_source_pair.path == file_url assert file_source_pair.file_source.id == "test1" - assert_realizes_contains( - file_sources, file_url, "DATA USE POLICIES", user_context=user_context - ) + assert_realizes_contains(file_sources, file_url, "DATA USE POLICIES", user_context=user_context)