Fix ftp import export using tasks

The FTP file source needs the user's context to access the ftp folder. It will also be required in case there are access restrictions on the target file source.

lib/galaxy/files/uris.py

@@ -47,13 +47,14 @@ def stream_url_to_file(
     file_sources: Optional["ConfiguredFileSources"] = None,
     prefix: str = "gx_file_stream",
     dir: Optional[str] = None,
+    user_context=None,
 ) -> str:
     temp_name: str
     if file_sources and file_sources.looks_like_uri(path):
         file_source_path = file_sources.get_file_source_path(path)
         with tempfile.NamedTemporaryFile(prefix=prefix, delete=False, dir=dir) as temp:
             temp_name = temp.name
-        file_source_path.file_source.realize_to(file_source_path.path, temp_name)
+        file_source_path.file_source.realize_to(file_source_path.path, temp_name, user_context=user_context)
     elif path.startswith("base64://"):
         with tempfile.NamedTemporaryFile(prefix=prefix, delete=False, dir=dir) as temp:
             temp_name = temp.name

lib/galaxy/managers/model_stores.py

@@ -3,7 +3,9 @@ from typing import Optional
 from galaxy import model
 from galaxy.exceptions import RequestParameterInvalidException
 from galaxy.jobs.manager import JobManager
+from galaxy.managers.context import ProvidesUserContext
 from galaxy.managers.histories import HistoryManager
+from galaxy.managers.users import UserManager
 from galaxy.model.scoped_session import galaxy_scoped_session
 from galaxy.model.store import (
     DirectoryModelExportStore,
@@ -32,6 +34,28 @@ from galaxy.web.short_term_storage import (
 )
 
 
+class ModelStoreUserContext(ProvidesUserContext):
+    def __init__(self, app: MinimalManagerApp, user: model.User) -> None:
+        self._app = app
+        self._user = user
+
+    @property
+    def app(self):
+        return self._app
+
+    @property
+    def url_builder(self):
+        raise NotImplementedError("URL builder not available in ModelStore context.")
+
+    def get_user(self):
+        return self._user
+
+    def set_user(self, user):
+        raise NotImplementedError("Cannot change user from ModelStore context.")
+
+    user = property(get_user, set_user)
+
+
 class ModelStoreManager:
     def __init__(
         self,
@@ -40,12 +64,14 @@ class ModelStoreManager:
         sa_session: galaxy_scoped_session,
         job_manager: JobManager,
         short_term_storage_monitor: ShortTermStorageMonitor,
+        user_manager: UserManager,
     ):
         self._app = app
         self._sa_session = sa_session
         self._job_manager = job_manager
         self._history_manager = history_manager
         self._short_term_storage_monitor = short_term_storage_monitor
+        self._user_manager = user_manager
 
     def setup_history_export_job(self, request: SetupHistoryExportJob):
         history_id = request.history_id
@@ -116,11 +142,13 @@ class ModelStoreManager:
         model_store_format = request.model_store_format
         export_files = "symlink" if request.include_files else None
         target_uri = request.target_uri
+        user_context = self._build_user_context(request.user.user_id)
         with model.store.get_export_store_factory(
             self._app,
             model_store_format,
             export_files=export_files,
             bco_export_options=self._bco_export_options(request),
+            user_context=user_context,
         )(target_uri) as export_store:
             invocation = self._sa_session.query(model.WorkflowInvocation).get(request.invocation_id)
             export_store.export_workflow_invocation(
@@ -142,9 +170,10 @@ class ModelStoreManager:
         model_store_format = request.model_store_format
         export_files = "symlink" if request.include_files else None
         target_uri = request.target_uri
-        with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(
-            target_uri
-        ) as export_store:
+        user_context = self._build_user_context(request.user.user_id)
+        with model.store.get_export_store_factory(
+            self._app, model_store_format, export_files=export_files, user_context=user_context
+        )(target_uri) as export_store:
             if request.content_type == HistoryContentType.dataset:
                 hda = self._sa_session.query(model.HistoryDatasetAssociation).get(request.content_id)
                 export_store.add_dataset(hda)
@@ -158,9 +187,10 @@ class ModelStoreManager:
         model_store_format = request.model_store_format
         export_files = "symlink" if request.include_files else None
         target_uri = request.target_uri
-        with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(
-            target_uri
-        ) as export_store:
+        user_context = self._build_user_context(request.user.user_id)
+        with model.store.get_export_store_factory(
+            self._app, model_store_format, export_files=export_files, user_context=user_context
+        )(target_uri) as export_store:
             history = self._history_manager.by_id(request.history_id)
             export_store.export_history(
                 history, include_hidden=request.include_hidden, include_deleted=request.include_deleted
@@ -175,17 +205,13 @@ class ModelStoreManager:
             history = self._sa_session.query(model.History).get(history_id)
         else:
             history = None
-        user_id = request.user.user_id
-        if user_id:
-            galaxy_user = self._sa_session.query(model.User).get(user_id)
-        else:
-            galaxy_user = None
+        user_context = self._build_user_context(request.user.user_id)
         model_import_store = source_to_import_store(
             request.source_uri,
             self._app,
-            galaxy_user,
             import_options,
             model_store_format=request.model_store_format,
+            user_context=user_context,
         )
         create_new_history = history is None and not request.for_library
         if create_new_history:
@@ -201,6 +227,11 @@ class ModelStoreManager:
             )
         return object_tracker
 
+    def _build_user_context(self, user_id: int):
+        user = self._user_manager.by_id(user_id)
+        user_context = ModelStoreUserContext(self._app, user)
+        return user_context
+
 
 def create_objects_from_store(
     app: MinimalManagerApp,
@@ -213,12 +244,13 @@ def create_objects_from_store(
         discarded_data=ImportDiscardedDataType.FORCE,
         allow_library_creation=for_library,
     )
+    user_context = ModelStoreUserContext(app, galaxy_user) if galaxy_user is not None else None
     model_import_store = source_to_import_store(
         payload.store_content_uri or payload.store_dict,
         app=app,
-        galaxy_user=galaxy_user,
         import_options=import_options,
         model_store_format=payload.model_store_format,
+        user_context=user_context,
     )
     create_new_history = history is None and not for_library
     if create_new_history:

lib/galaxy/model/store/__init__.py

@@ -50,7 +50,10 @@ from galaxy.exceptions import (
     ObjectNotFound,
     RequestParameterInvalidException,
 )
-from galaxy.files import ConfiguredFileSources
+from galaxy.files import (
+    ConfiguredFileSources,
+    ProvidesUserFileSourcesUserContext,
+)
 from galaxy.files.uris import stream_url_to_file
 from galaxy.model.mapping import GalaxyModelMapping
 from galaxy.model.metadata import MetadataCollection
@@ -1812,6 +1815,7 @@ class DirectoryModelExportStore(ModelExportStore):
         export_files: Optional[str] = None,
         strip_metadata_files: bool = True,
         serialize_jobs: bool = True,
+        user_context=None,
     ) -> None:
         """
         :param export_directory: path to export directory. Will be created if it does not exist.
@@ -1836,6 +1840,7 @@ class DirectoryModelExportStore(ModelExportStore):
             sessionless = True
             security = IdEncodingHelper(id_secret="randomdoesntmatter")
 
+        self.user_context = ProvidesUserFileSourcesUserContext(user_context)
         self.file_sources = file_sources
         self.serialize_jobs = serialize_jobs
         self.sessionless = sessionless
@@ -2500,7 +2505,7 @@ class BcoModelExportStore(WorkflowInvocationOnlyExportStore):
             file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)
             file_source = file_source_path.file_source
             assert os.path.exists(self.out_file)
-            file_source.write_from(file_source_path.path, self.out_file)
+            file_source.write_from(file_source_path.path, self.out_file, user_context=self.user_context)
 
     def _core_biocompute_object_and_object_id(self) -> Tuple[BioComputeObjectCore, str]:
         assert self.app  # need app.security to do anything...
@@ -2725,7 +2730,7 @@ class ROCrateArchiveModelExportStore(DirectoryModelExportStore, WriteCrates):
             file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)
             file_source = file_source_path.file_source
             assert os.path.exists(rval), rval
-            file_source.write_from(file_source_path.path, rval)
+            file_source.write_from(file_source_path.path, rval, user_context=self.user_context)
         shutil.rmtree(self.temp_output_dir)
 
 
@@ -2756,7 +2761,7 @@ class TarModelExportStore(DirectoryModelExportStore):
             file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)
             file_source = file_source_path.file_source
             assert os.path.exists(self.out_file)
-            file_source.write_from(file_source_path.path, self.out_file)
+            file_source.write_from(file_source_path.path, self.out_file, user_context=self.user_context)
         shutil.rmtree(self.temp_output_dir)
 
 
@@ -2799,12 +2804,16 @@ class BagArchiveModelExportStore(BagDirectoryModelExportStore):
             file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)
             file_source = file_source_path.file_source
             assert os.path.exists(rval)
-            file_source.write_from(file_source_path.path, rval)
+            file_source.write_from(file_source_path.path, rval, user_context=self.user_context)
         shutil.rmtree(self.temp_output_dir)
 
 
 def get_export_store_factory(
-    app, download_format: str, export_files=None, bco_export_options: Optional[BcoExportOptions] = None
+    app,
+    download_format: str,
+    export_files=None,
+    bco_export_options: Optional[BcoExportOptions] = None,
+    user_context=None,
 ) -> Callable[[StrPath], ModelExportStore]:
     export_store_class: Union[
         Type[TarModelExportStore],
@@ -2816,6 +2825,7 @@ def get_export_store_factory(
         "app": app,
         "export_files": export_files,
         "serialize_dataset_objects": False,
+        "user_context": user_context,
     }
     if download_format in ["tar.gz", "tgz"]:
         export_store_class = TarModelExportStore
@@ -2871,10 +2881,11 @@ def imported_store_for_metadata(
 def source_to_import_store(
     source: Union[str, dict],
     app: StoreAppProtocol,
-    galaxy_user: Optional[model.User],
     import_options: Optional[ImportOptions],
     model_store_format: Optional[ModelStoreFormat] = None,
+    user_context=None,
 ) -> ModelImportStore:
+    galaxy_user = user_context.user if user_context else None
     if isinstance(source, dict):
         if model_store_format is not None:
             raise Exception(
@@ -2893,7 +2904,10 @@ def source_to_import_store(
         if source_uri.startswith("file://"):
             source_uri = source_uri[len("file://") :]
         if "://" in source_uri:
-            source_uri = stream_url_to_file(source_uri, app.file_sources, prefix="gx_import_model_store")
+            user_context = ProvidesUserFileSourcesUserContext(user_context)
+            source_uri = stream_url_to_file(
+                source_uri, app.file_sources, prefix="gx_import_model_store", user_context=user_context
+            )
             delete = True
         target_path = source_uri
         if target_path.endswith(".json"):