From 62dae772ec79eafa848a486993df7e5088437d92 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 9 Mar 2022 13:51:00 -0500 Subject: [PATCH] Add model stores manager. Reused for a bunch of things in the deferred data branch but it does simplify the tasks file for now and move more logic into managers - which I think is a better place. --- lib/galaxy/celery/tasks.py | 22 ++++------------ lib/galaxy/managers/model_stores.py | 38 ++++++++++++++++++++++++++++ lib/galaxy/schema/tasks.py | 10 ++++++++ lib/galaxy/tools/imp_exp/__init__.py | 27 +++++++++----------- 4 files changed, 65 insertions(+), 32 deletions(-) create mode 100644 lib/galaxy/managers/model_stores.py create mode 100644 lib/galaxy/schema/tasks.py diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 08f20deadcf..181e00e05eb 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,10 +1,10 @@ from galaxy import model from galaxy.celery import galaxy_task -from galaxy.jobs.manager import JobManager from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager +from galaxy.managers.model_stores import ModelStoreManager from galaxy.model.scoped_session import galaxy_scoped_session -from galaxy.structured_app import MinimalManagerApp +from galaxy.schema.tasks import SetupHistoryExportJob from galaxy.util.custom_logging import get_logger log = get_logger(__name__) @@ -42,22 +42,10 @@ def set_metadata( @galaxy_task(ignore_result=True, action="setting up export history job") def export_history( - app: MinimalManagerApp, - sa_session: galaxy_scoped_session, - job_manager: JobManager, - store_directory: str, - history_id: int, - job_id: int, - include_hidden=False, - include_deleted=False, + model_store_manager: ModelStoreManager, + request: SetupHistoryExportJob, ): - history = sa_session.query(model.History).get(history_id) - with model.store.DirectoryModelExportStore(store_directory, app=app, export_files="symlink") as export_store: - export_store.export_history(history, include_hidden=include_hidden, include_deleted=include_deleted) - job = sa_session.query(model.Job).filter_by(id=job_id).one() - job.state = model.Job.states.NEW - sa_session.flush() - job_manager.enqueue(job) + model_store_manager.setup_history_export_job(request) @galaxy_task(action="pruning history audit table") diff --git a/lib/galaxy/managers/model_stores.py b/lib/galaxy/managers/model_stores.py new file mode 100644 index 00000000000..eeab299ee27 --- /dev/null +++ b/lib/galaxy/managers/model_stores.py @@ -0,0 +1,38 @@ +from galaxy import model +from galaxy.app import MinimalManagerApp +from galaxy.jobs.manager import JobManager +from galaxy.managers.histories import HistoryManager +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.schema.tasks import SetupHistoryExportJob + + +class ModelStoreManager: + def __init__( + self, + app: MinimalManagerApp, + history_manager: HistoryManager, + sa_session: galaxy_scoped_session, + job_manager: JobManager, + ): + self._app = app + self._sa_session = sa_session + self._job_manager = job_manager + self._history_manager = history_manager + + def setup_history_export_job(self, request: SetupHistoryExportJob): + history_id = request.history_id + job_id = request.job_id + include_hidden = request.include_hidden + include_deleted = request.include_deleted + store_directory = request.store_directory + + history = self._sa_session.query(model.History).get(history_id) + # symlink files on export, on worker files will tarred up in a dereferenced manner. + with model.store.DirectoryModelExportStore( + store_directory, app=self._app, export_files="symlink" + ) as export_store: + export_store.export_history(history, include_hidden=include_hidden, include_deleted=include_deleted) + job = self._sa_session.query(model.Job).get(job_id) + job.state = model.Job.states.NEW + self._sa_session.flush() + self._job_manager.enqueue(job) diff --git a/lib/galaxy/schema/tasks.py b/lib/galaxy/schema/tasks.py new file mode 100644 index 00000000000..71b04ec320a --- /dev/null +++ b/lib/galaxy/schema/tasks.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + + +class SetupHistoryExportJob(BaseModel): + history_id: int + job_id: int + store_directory: str + include_files: bool + include_hidden: bool + include_deleted: bool diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index df125fc5dda..d07af19fee5 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,6 +5,7 @@ import shutil from galaxy import model from galaxy.model import store +from galaxy.schema.tasks import SetupHistoryExportJob from galaxy.util.path import external_chown log = logging.getLogger(__name__) @@ -98,23 +99,19 @@ class JobExportHistoryArchiveWrapper: Perform setup for job to export a history into an archive. """ app = self.app - + # TODO: prevent circular import here... from galaxy.celery.tasks import export_history + request = SetupHistoryExportJob( + history_id=history.id, + job_id=self.job_id, + store_directory=store_directory, + include_files=True, + include_hidden=include_hidden, + include_deleted=include_deleted, + ) if app.config.enable_celery_tasks: # symlink files on export, on worker files will tarred up in a dereferenced manner. - export_history.delay( - store_directory=store_directory, - history_id=history.id, - job_id=self.job_id, - include_hidden=include_hidden, - include_deleted=include_deleted, - ) + export_history.delay(request=request) else: - export_history( - store_directory=store_directory, - history_id=history.id, - job_id=self.job_id, - include_hidden=include_hidden, - include_deleted=include_deleted, - ) + export_history(request=request)