From aa73b52a201d8ca8bf03e3b0237459a4cec27887 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 Dec 2021 11:53:48 +0100 Subject: [PATCH 1/2] Use yield_per option to limit amount of ORM objects loaded into memory I've applied it only to non-legacy code where we don't need the entire list up-front. This might help with > I have an issue with the job handlers from a few days. Randomly one of them gets trapped in a death loop. It starts to monitor jobs, the memory ramps up until the systemd limit of 12GB and then is killed by sytemd report by @gmauro on usegalaxy.eu --- lib/galaxy/jobs/handler.py | 4 ++-- lib/galaxy/model/__init__.py | 1 + lib/galaxy/webapps/galaxy/api/jobs.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 90d32966b68..d1492150f8f 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -237,11 +237,11 @@ class JobHandlerQueue(Monitors): .outerjoin(model.User) \ .filter(model.Job.state.in_(in_list) & (model.Job.handler == self.app.config.server_name) - & or_((model.Job.user_id == null()), (model.User.active == true()))).all() + & or_((model.Job.user_id == null()), (model.User.active == true()))).yield_per(model.YIELD_PER_ROWS) else: jobs_at_startup = self.sa_session.query(model.Job).enable_eagerloads(False) \ .filter(model.Job.state.in_(in_list) - & (model.Job.handler == self.app.config.server_name)).all() + & (model.Job.handler == self.app.config.server_name)).yield_per(model.YIELD_PER_ROWS) for job in jobs_at_startup: if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 072cb37f05e..e2d1544cb93 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -135,6 +135,7 @@ JOB_METRIC_PRECISION = 26 JOB_METRIC_SCALE = 7 # Tags that get automatically propagated from inputs to outputs when running jobs. AUTO_PROPAGATED_TAGS = ["name"] +YIELD_PER_ROWS = 100 if TYPE_CHECKING: diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index e856a16450b..a63f2efcdae 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -204,7 +204,7 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): query = query.limit(limit) out = [] - for job in query.all(): + for job in query.yield_per(model.YIELD_PER_ROWS): job_dict = job.to_dict(view, system_details=is_admin) j = self.encode_all_ids(trans, job_dict, True) if view == 'admin_job_list': From 9e52fb6ac7a45a06839e4efd878f1020e96fc82f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 Dec 2021 12:08:07 +0100 Subject: [PATCH 2/2] Disable eagerload of job pja relationship --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e2d1544cb93..ad671eaf389 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -916,7 +916,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): back_populates='job', lazy=True) output_dataset_collections = relationship('JobToImplicitOutputDatasetCollectionAssociation', back_populates='job', lazy=True) - post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=False) + post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=True) input_library_datasets = relationship('JobToInputLibraryDatasetAssociation', back_populates='job') output_library_datasets = relationship('JobToOutputLibraryDatasetAssociation',