From e73731f5528a08735a6b65bf419a6f7986c1c771 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Feb 2018 22:05:46 -0500 Subject: [PATCH] Optimize database interaction for user workflow list. Don't LEFT OUTER JOIN workflow against its steps to produce step count - this causes a lot of duplicated workflow data and fetches step data not needed to just produce a step count. Tweak pre-fetching for workflows shared with user also. --- lib/galaxy/model/mapping.py | 8 +++++++- lib/galaxy/webapps/galaxy/api/workflows.py | 12 +++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f2ffed6188b..2fd9c7ab9ee 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -14,6 +14,7 @@ from sqlalchemy import ( desc, false, ForeignKey, + func, Integer, MetaData, not_, @@ -2115,7 +2116,12 @@ mapper(model.Workflow, model.Workflow.table, properties=dict( primaryjoin=((model.Workflow.table.c.id == model.WorkflowStep.table.c.workflow_id)), order_by=asc(model.WorkflowStep.table.c.order_index), cascade="all, delete-orphan", - lazy=False) + lazy=False), + step_count=column_property( + select([func.count(model.WorkflowStep.table.c.id)]).where(model.Workflow.table.c.id == model.WorkflowStep.table.c.workflow_id), + deferred=True + ) + )) mapper(model.WorkflowStep, model.WorkflowStep.table, properties=dict( diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 6e6233173ce..c43d46354bf 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -7,6 +7,7 @@ import logging from six.moves.urllib.parse import unquote_plus from sqlalchemy import desc, false, or_, true +from sqlalchemy.orm import eagerload from galaxy import ( exceptions, @@ -125,7 +126,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn user = trans.get_user() if show_published: filter1 = or_(filter1, (trans.app.model.StoredWorkflow.published == true())) - for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).filter( + for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).options( + eagerload("latest_workflow").undefer("step_count").lazyload("steps")).filter( filter1, trans.app.model.StoredWorkflow.table.c.deleted == false()).order_by( desc(trans.app.model.StoredWorkflow.table.c.update_time)).all(): @@ -133,15 +135,15 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn encoded_id = trans.security.encode_id(wf.id) item['url'] = url_for('workflow', id=encoded_id) item['owner'] = wf.user.username - item['number_of_steps'] = len(wf.latest_workflow.steps) + item['number_of_steps'] = wf.latest_workflow.step_count item['show_in_tool_panel'] = False for x in user.stored_workflow_menu_entries: if x.stored_workflow_id == wf.id: item['show_in_tool_panel'] = True break rval.append(item) - for wf_sa in trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by( - user=trans.user).join('stored_workflow').filter( + for wf_sa in trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).options( + eagerload("stored_workflow").joinedload("latest_workflow").undefer("step_count").lazyload("steps")).filter_by(user=trans.user).filter( trans.app.model.StoredWorkflow.deleted == false()).order_by( desc(trans.app.model.StoredWorkflow.update_time)).all(): item = wf_sa.stored_workflow.to_dict(value_mapper={'id': trans.security.encode_id}) @@ -149,7 +151,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn item['url'] = url_for('workflow', id=encoded_id) item['slug'] = wf_sa.stored_workflow.slug item['owner'] = wf_sa.stored_workflow.user.username - item['number_of_steps'] = len(wf_sa.stored_workflow.latest_workflow.steps) + item['number_of_steps'] = wf_sa.stored_workflow.latest_workflow.step_count item['show_in_tool_panel'] = False for x in user.stored_workflow_menu_entries: if x.stored_workflow_id == wf_sa.id: