Merge pull request #13076 from mvdbeek/yield_per

[21.09] Use yield_per option to limit amount of ORM objects loaded into memory

lib/galaxy/jobs/handler.py

@@ -237,11 +237,11 @@ class JobHandlerQueue(Monitors):
                 .outerjoin(model.User) \
                 .filter(model.Job.state.in_(in_list)
                         & (model.Job.handler == self.app.config.server_name)
-                        & or_((model.Job.user_id == null()), (model.User.active == true()))).all()
+                        & or_((model.Job.user_id == null()), (model.User.active == true()))).yield_per(model.YIELD_PER_ROWS)
         else:
             jobs_at_startup = self.sa_session.query(model.Job).enable_eagerloads(False) \
                 .filter(model.Job.state.in_(in_list)
-                        & (model.Job.handler == self.app.config.server_name)).all()
+                        & (model.Job.handler == self.app.config.server_name)).yield_per(model.YIELD_PER_ROWS)
 
         for job in jobs_at_startup:
             if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True):

lib/galaxy/model/__init__.py

@@ -135,6 +135,7 @@ JOB_METRIC_PRECISION = 26
 JOB_METRIC_SCALE = 7
 # Tags that get automatically propagated from inputs to outputs when running jobs.
 AUTO_PROPAGATED_TAGS = ["name"]
+YIELD_PER_ROWS = 100
 
 
 if TYPE_CHECKING:
@@ -915,7 +916,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
         back_populates='job', lazy=True)
     output_dataset_collections = relationship('JobToImplicitOutputDatasetCollectionAssociation',
         back_populates='job', lazy=True)
-    post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=False)
+    post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=True)
     input_library_datasets = relationship('JobToInputLibraryDatasetAssociation',
         back_populates='job')
     output_library_datasets = relationship('JobToOutputLibraryDatasetAssociation',

lib/galaxy/webapps/galaxy/api/jobs.py

@@ -204,7 +204,7 @@ class JobController(BaseGalaxyAPIController, UsesVisualizationMixin):
         query = query.limit(limit)
 
         out = []
-        for job in query.all():
+        for job in query.yield_per(model.YIELD_PER_ROWS):
             job_dict = job.to_dict(view, system_details=is_admin)
             j = self.encode_all_ids(trans, job_dict, True)
             if view == 'admin_job_list':