From 80a41f32744d2d76b74973663cd6d751daf8e8a5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Feb 2018 16:43:27 -0500 Subject: [PATCH] Optimize public grid database interactions. For all four grid types (histories, viz, workflows, and pages), this computes rating average on the server and prefetches any annotations. This eliminates at least two extra queries for response element in the grid. See comment in the history controller for why I am fairly confident this is a good idea for annotations but tags are less obvious - these grids still go back to the postgres server multiple times per rendered item to render tags. I'm confident we should either subqueryload, joinedload, or upgrade to sqlalchemy 2.2 and selectinload (http://docs.sqlalchemy.org/en/latest/orm/loading_relationships.html#sqlalchemy.orm.selectinload) the tags as well - but I'm not sure which without being able to hack on a usegalaxy.org. This also brings in less of the user model (only username instead of all of it) to reduce over-the-wire transmission of unneeded data from postgres to Galaxy. For workflows we were LEFT OUTER JOIN-ing on the steps of the latest workflow - so we were bringing back a lot of extra rows for data that was completely unused. In light of this, it makes perfect sense to me why published workflows were the slowest of these and I suspect they will all be equally performant after this change (modulo the number of rows in the tables and the number of rows rendered). --- lib/galaxy/model/mapping.py | 25 ++++++++++++++++--- lib/galaxy/web/framework/helpers/grids.py | 7 +++++- .../webapps/galaxy/controllers/history.py | 18 ++++++++++--- lib/galaxy/webapps/galaxy/controllers/page.py | 5 ++-- .../galaxy/controllers/visualization.py | 5 ++-- .../webapps/galaxy/controllers/workflow.py | 8 +++--- 6 files changed, 53 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f2ffed6188b..d3531ea8fa8 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -14,6 +14,7 @@ from sqlalchemy import ( desc, false, ForeignKey, + func, Integer, MetaData, not_, @@ -1569,7 +1570,11 @@ mapper(model.History, model.History.table, properties=dict( backref="histories"), ratings=relation(model.HistoryRatingAssociation, order_by=model.HistoryRatingAssociation.table.c.id, - backref="histories") + backref="histories"), + average_rating=column_property( + select([func.avg(model.HistoryRatingAssociation.table.c.rating)]).where(model.HistoryRatingAssociation.table.c.history_id == model.History.table.c.id), + deferred=True + ) )) # Set up proxy so that @@ -2178,7 +2183,11 @@ mapper(model.StoredWorkflow, model.StoredWorkflow.table, properties=dict( backref="stored_workflows"), ratings=relation(model.StoredWorkflowRatingAssociation, order_by=model.StoredWorkflowRatingAssociation.table.c.id, - backref="stored_workflows") + backref="stored_workflows"), + average_rating=column_property( + select([func.avg(model.StoredWorkflowRatingAssociation.table.c.rating)]).where(model.StoredWorkflowRatingAssociation.table.c.stored_workflow_id == model.StoredWorkflow.table.c.id), + deferred=True + ) )) # Set up proxy so that @@ -2310,7 +2319,11 @@ mapper(model.Page, model.Page.table, properties=dict( backref="pages"), ratings=relation(model.PageRatingAssociation, order_by=model.PageRatingAssociation.table.c.id, - backref="pages") + backref="pages"), + average_rating=column_property( + select([func.avg(model.PageRatingAssociation.table.c.rating)]).where(model.PageRatingAssociation.table.c.page_id == model.Page.table.c.id), + deferred=True + ) )) # Set up proxy so that @@ -2342,7 +2355,11 @@ mapper(model.Visualization, model.Visualization.table, properties=dict( backref="visualizations"), ratings=relation(model.VisualizationRatingAssociation, order_by=model.VisualizationRatingAssociation.table.c.id, - backref="visualizations") + backref="visualizations"), + average_rating=column_property( + select([func.avg(model.VisualizationRatingAssociation.table.c.rating)]).where(model.VisualizationRatingAssociation.table.c.visualization_id == model.Visualization.table.c.id), + deferred=True + ) )) # Set up proxy so that diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 69354b891d8..2c6da1988c6 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -573,7 +573,12 @@ class CommunityRatingColumn(GridColumn, UsesItemRatings): """ Column that displays community ratings for an item. """ def get_value(self, trans, grid, item): - ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, item, webapp_model=trans.model) + if not hasattr(item, "average_rating"): + # No prefetched column property, generate it on the fly. + ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, item, webapp_model=trans.model) + else: + ave_item_rating = item.average_rating + num_ratings = 2 # just used for pluralization return trans.fill_template("tool_shed_rating.mako", ave_item_rating=ave_item_rating, num_ratings=num_ratings, diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 64772493454..8e34e557f2c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -5,7 +5,7 @@ from markupsafe import escape from six import string_types from six.moves.urllib.parse import unquote_plus from sqlalchemy import and_, false, func, null, true -from sqlalchemy.orm import eagerload, eagerload_all +from sqlalchemy.orm import eagerload, eagerload_all, undefer import galaxy.util from galaxy import exceptions @@ -209,8 +209,20 @@ class HistoryAllPublishedGrid(grids.Grid): operations = [] def build_initial_query(self, trans, **kwargs): - # Join so that searching history.user makes sense. - return trans.sa_session.query(self.model_class).join(model.User.table) + # TODO: Tags are still loaded one at a time, consider doing this all at once: + # - eagerload would keep everything in one query but would explode the number of rows and potentially + # result in unneeded info transferred over the wire. + # - subqueryload("tags").subqueryload("tag") would probably be better under postgres but I'd + # like some performance data against a big database first - might cause problems? + + # - Pull down only username from associated User table since that is all that is used + # (can be used during search). Need join in addition to the eagerload since it is used in + # the .count() query which doesn't respect the eagerload options (could eliminate this with #5523). + # - Undefer average_rating column to prevent loading individual ratings per-history. + # - Eager load annotations - this causes a left join which might be inefficient if there were + # potentially many items per history (like if joining HDAs for instance) but there should only + # be at most one so this is fine. + return trans.sa_session.query(self.model_class).join("user").options(eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating")) def apply_query_filter(self, trans, query, **kwargs): # A public history is published, has a slug, and is not deleted. diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 8f9ff09a41d..2cff6220813 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -2,6 +2,7 @@ from json import loads from markupsafe import escape from sqlalchemy import and_, desc, false, true +from sqlalchemy.orm import eagerload, undefer from galaxy import managers, model, util, web from galaxy.model.item_attrs import UsesItemRatings @@ -82,8 +83,8 @@ class PageAllPublishedGrid(grids.Grid): ) def build_initial_query(self, trans, **kwargs): - # Join so that searching history.user makes sense. - return trans.sa_session.query(self.model_class).join(model.User.table) + # See optimization description comments and TODO for tags in matching public histories query. + return trans.sa_session.query(self.model_class).join("user").options(eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating")) def apply_query_filter(self, trans, query, **kwargs): return query.filter(self.model_class.deleted == false()).filter(self.model_class.published == true()) diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index 4e0aa904e2d..0232494646e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -12,6 +12,7 @@ from paste.httpexceptions import ( ) from six import string_types from sqlalchemy import and_, desc, false, or_, true +from sqlalchemy.orm import eagerload, undefer from galaxy import managers, model, util, web from galaxy.datatypes.interval import Bed @@ -212,8 +213,8 @@ class VisualizationAllPublishedGrid(grids.Grid): ) def build_initial_query(self, trans, **kwargs): - # Join so that searching history.user makes sense. - return trans.sa_session.query(self.model_class).join(model.User.table) + # See optimization description comments and TODO for tags in matching public histories query. + return trans.sa_session.query(self.model_class).join("user").options(eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating")) def apply_query_filter(self, trans, query, **kwargs): return query.filter(self.model_class.deleted == false()).filter(self.model_class.published == true()) diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index e5b9ba020f6..9becce4c39f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -10,7 +10,7 @@ import requests from markupsafe import escape from six.moves.http_client import HTTPConnection from sqlalchemy import and_ -from sqlalchemy.orm import joinedload +from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer from sqlalchemy.sql import expression from galaxy import ( @@ -138,8 +138,10 @@ class StoredWorkflowAllPublishedGrid(grids.Grid): ] def build_initial_query(self, trans, **kwargs): - # Join so that searching stored_workflow.user makes sense. - return trans.sa_session.query(self.model_class).join(model.User.table) + # See optimization description comments and TODO for tags in matching public histories query. + # In addition to that - be sure to lazyload the latest_workflow - it isn't needed and it causes all + # of its steps to be eagerly loaded. + return trans.sa_session.query(self.model_class).join("user").options(lazyload("latest_workflow"), eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating")) def apply_query_filter(self, trans, query, **kwargs): # A public workflow is published, has a slug, and is not deleted.