Merge pull request #5540 from jmchilton/dev_merge_0

Merge release_18.01 into dev.

client/galaxy/scripts/onload.js

@@ -179,18 +179,20 @@ $(document).ready(() => {
 
     function onloadWebhooks() {
         if (Galaxy.root !== undefined) {
-            // Load all webhooks with the type 'onload'
-            Webhooks.load({
-                type: "onload",
-                callback: function(webhooks) {
-                    webhooks.each(model => {
-                        var webhook = model.toJSON();
-                        if (webhook.activate && webhook.script) {
-                            Utils.appendScriptStyle(webhook);
-                        }
-                    });
-                }
-            });
+            if (Galaxy.config.enable_webhooks) {
+                // Load all webhooks with the type 'onload'
+                Webhooks.load({
+                    type: "onload",
+                    callback: function(webhooks) {
+                        webhooks.each(model => {
+                            var webhook = model.toJSON();
+                            if (webhook.activate && webhook.script) {
+                                Utils.appendScriptStyle(webhook);
+                            }
+                        });
+                    }
+                });
+            }
         } else {
             setTimeout(onloadWebhooks, 100);
         }

lib/galaxy/datatypes/binary.py

@@ -12,6 +12,7 @@ import sys
 import tarfile
 import tempfile
 import zipfile
+from collections import OrderedDict
 from json import dumps
 
 import h5py
@@ -234,10 +235,10 @@ class BamNative(Binary):
             # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
             dataset.metadata.reference_names = list(bam_file.references)
             dataset.metadata.reference_lengths = list(bam_file.lengths)
-            dataset.metadata.bam_header = bam_file.header
+            dataset.metadata.bam_header = OrderedDict((k, v) for k, v in bam_file.header.items())
             dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
-            dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
-            dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
+            dataset.metadata.sort_order = dataset.metadata.bam_header.get('HD', {}).get('SO', None)
+            dataset.metadata.bam_version = dataset.metadata.bam_header.get('HD', {}).get('VN', None)
         except Exception:
             # Per Dan, don't log here because doing so will cause datasets that
             # fail metadata to end in the error state
@@ -383,25 +384,12 @@ class Bam(BamNative):
 
     def set_meta(self, dataset, overwrite=True, **kwd):
         # These metadata values are not accessible by users, always overwrite
+        super(Bam, self).set_meta(dataset=dataset, overwrite=overwrite, **kwd)
         index_file = dataset.metadata.bam_index
         if not index_file:
             index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset)
         pysam.index(dataset.file_name, index_file.file_name)
         dataset.metadata.bam_index = index_file
-        # Now use pysam with BAI index to determine additional metadata
-        try:
-            bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name)
-            # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
-            dataset.metadata.reference_names = list(bam_file.references)
-            dataset.metadata.reference_lengths = list(bam_file.lengths)
-            dataset.metadata.bam_header = bam_file.header
-            dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
-            dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
-            dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
-        except Exception:
-            # Per Dan, don't log here because doing so will cause datasets that
-            # fail metadata to end in the error state
-            pass
 
     def sniff(self, file_name):
         return super(Bam, self).sniff(file_name) and not self.dataset_content_needs_grooming(file_name)

lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml

@@ -1,4 +1,4 @@
-<tool id="CONVERTER_bam_native_to_bam" name="Convert BAM native to BAM" version="1.0.0" hidden="true">
+<tool id="CONVERTER_bam_native_to_bam" name="Convert BAM native to BAM" version="1.0.0" hidden="true" profile="18.01">
     <!--  <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <requirements>
         <requirement type="package" version="1.6">samtools</requirement>

lib/galaxy/datatypes/converters/sam_to_bam_native.xml

@@ -1,4 +1,4 @@
-<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0">
+<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0" profile="18.01">
     <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <requirements>
         <requirement type="package" version="1.6">samtools</requirement>

lib/galaxy/dependencies/pinned-requirements.txt

@@ -11,7 +11,7 @@ mercurial==3.7.3; python_version < '3.0'
 pycrypto==2.6.1
 uWSGI==2.0.15
 # Flexible BAM index naming is new to core pysam
-pysam>=0.13
+pysam==0.14
 
 # Install python_lzo if you want to support indexed access to lzo-compressed
 # locally cached maf files via bx-python

lib/galaxy/model/mapping.py

@@ -14,6 +14,7 @@ from sqlalchemy import (
     desc,
     false,
     ForeignKey,
+    func,
     Integer,
     MetaData,
     not_,
@@ -1569,7 +1570,11 @@ mapper(model.History, model.History.table, properties=dict(
         backref="histories"),
     ratings=relation(model.HistoryRatingAssociation,
         order_by=model.HistoryRatingAssociation.table.c.id,
-        backref="histories")
+        backref="histories"),
+    average_rating=column_property(
+        select([func.avg(model.HistoryRatingAssociation.table.c.rating)]).where(model.HistoryRatingAssociation.table.c.history_id == model.History.table.c.id),
+        deferred=True
+    )
 ))
 
 # Set up proxy so that
@@ -2115,7 +2120,12 @@ mapper(model.Workflow, model.Workflow.table, properties=dict(
         primaryjoin=((model.Workflow.table.c.id == model.WorkflowStep.table.c.workflow_id)),
         order_by=asc(model.WorkflowStep.table.c.order_index),
         cascade="all, delete-orphan",
-        lazy=False)
+        lazy=False),
+    step_count=column_property(
+        select([func.count(model.WorkflowStep.table.c.id)]).where(model.Workflow.table.c.id == model.WorkflowStep.table.c.workflow_id),
+        deferred=True
+    )
+
 ))
 
 mapper(model.WorkflowStep, model.WorkflowStep.table, properties=dict(
@@ -2178,7 +2188,11 @@ mapper(model.StoredWorkflow, model.StoredWorkflow.table, properties=dict(
         backref="stored_workflows"),
     ratings=relation(model.StoredWorkflowRatingAssociation,
         order_by=model.StoredWorkflowRatingAssociation.table.c.id,
-        backref="stored_workflows")
+        backref="stored_workflows"),
+    average_rating=column_property(
+        select([func.avg(model.StoredWorkflowRatingAssociation.table.c.rating)]).where(model.StoredWorkflowRatingAssociation.table.c.stored_workflow_id == model.StoredWorkflow.table.c.id),
+        deferred=True
+    )
 ))
 
 # Set up proxy so that
@@ -2310,7 +2324,11 @@ mapper(model.Page, model.Page.table, properties=dict(
         backref="pages"),
     ratings=relation(model.PageRatingAssociation,
         order_by=model.PageRatingAssociation.table.c.id,
-        backref="pages")
+        backref="pages"),
+    average_rating=column_property(
+        select([func.avg(model.PageRatingAssociation.table.c.rating)]).where(model.PageRatingAssociation.table.c.page_id == model.Page.table.c.id),
+        deferred=True
+    )
 ))
 
 # Set up proxy so that
@@ -2342,7 +2360,11 @@ mapper(model.Visualization, model.Visualization.table, properties=dict(
         backref="visualizations"),
     ratings=relation(model.VisualizationRatingAssociation,
         order_by=model.VisualizationRatingAssociation.table.c.id,
-        backref="visualizations")
+        backref="visualizations"),
+    average_rating=column_property(
+        select([func.avg(model.VisualizationRatingAssociation.table.c.rating)]).where(model.VisualizationRatingAssociation.table.c.visualization_id == model.Visualization.table.c.id),
+        deferred=True
+    )
 ))
 
 # Set up proxy so that

lib/galaxy/web/framework/helpers/grids.py

@@ -573,7 +573,12 @@ class CommunityRatingColumn(GridColumn, UsesItemRatings):
     """ Column that displays community ratings for an item. """
 
     def get_value(self, trans, grid, item):
-        ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, item, webapp_model=trans.model)
+        if not hasattr(item, "average_rating"):
+            # No prefetched column property, generate it on the fly.
+            ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, item, webapp_model=trans.model)
+        else:
+            ave_item_rating = item.average_rating
+            num_ratings = 2  # just used for pluralization
         return trans.fill_template("tool_shed_rating.mako",
                                    ave_item_rating=ave_item_rating,
                                    num_ratings=num_ratings,

lib/galaxy/webapps/galaxy/api/workflows.py

@@ -7,6 +7,7 @@ import logging
 
 from six.moves.urllib.parse import unquote_plus
 from sqlalchemy import desc, false, or_, true
+from sqlalchemy.orm import eagerload
 
 from galaxy import (
     exceptions,
@@ -125,7 +126,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
         user = trans.get_user()
         if show_published:
             filter1 = or_(filter1, (trans.app.model.StoredWorkflow.published == true()))
-        for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).filter(
+        for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).options(
+                eagerload("latest_workflow").undefer("step_count").lazyload("steps")).filter(
                 filter1, trans.app.model.StoredWorkflow.table.c.deleted == false()).order_by(
                 desc(trans.app.model.StoredWorkflow.table.c.update_time)).all():
 
@@ -133,15 +135,15 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
             encoded_id = trans.security.encode_id(wf.id)
             item['url'] = url_for('workflow', id=encoded_id)
             item['owner'] = wf.user.username
-            item['number_of_steps'] = len(wf.latest_workflow.steps)
+            item['number_of_steps'] = wf.latest_workflow.step_count
             item['show_in_tool_panel'] = False
             for x in user.stored_workflow_menu_entries:
                 if x.stored_workflow_id == wf.id:
                     item['show_in_tool_panel'] = True
                     break
             rval.append(item)
-        for wf_sa in trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by(
-                user=trans.user).join('stored_workflow').filter(
+        for wf_sa in trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).options(
+                eagerload("stored_workflow").joinedload("latest_workflow").undefer("step_count").lazyload("steps")).filter_by(user=trans.user).filter(
                 trans.app.model.StoredWorkflow.deleted == false()).order_by(
                 desc(trans.app.model.StoredWorkflow.update_time)).all():
             item = wf_sa.stored_workflow.to_dict(value_mapper={'id': trans.security.encode_id})
@@ -149,7 +151,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
             item['url'] = url_for('workflow', id=encoded_id)
             item['slug'] = wf_sa.stored_workflow.slug
             item['owner'] = wf_sa.stored_workflow.user.username
-            item['number_of_steps'] = len(wf_sa.stored_workflow.latest_workflow.steps)
+            item['number_of_steps'] = wf_sa.stored_workflow.latest_workflow.step_count
             item['show_in_tool_panel'] = False
             for x in user.stored_workflow_menu_entries:
                 if x.stored_workflow_id == wf_sa.id:

lib/galaxy/webapps/galaxy/controllers/history.py

@@ -4,7 +4,7 @@ from markupsafe import escape
 from six import string_types
 from six.moves.urllib.parse import unquote_plus
 from sqlalchemy import and_, false, func, null, true
-from sqlalchemy.orm import eagerload, eagerload_all
+from sqlalchemy.orm import eagerload, eagerload_all, undefer
 
 import galaxy.util
 from galaxy import exceptions
@@ -208,8 +208,20 @@ class HistoryAllPublishedGrid(grids.Grid):
     operations = []
 
     def build_initial_query(self, trans, **kwargs):
-        # Join so that searching history.user makes sense.
-        return trans.sa_session.query(self.model_class).join(model.User.table)
+        # TODO: Tags are still loaded one at a time, consider doing this all at once:
+        # - eagerload would keep everything in one query but would explode the number of rows and potentially
+        #   result in unneeded info transferred over the wire.
+        # - subqueryload("tags").subqueryload("tag") would probably be better under postgres but I'd
+        #   like some performance data against a big database first - might cause problems?
+
+        # - Pull down only username from associated User table since that is all that is used
+        #   (can be used during search). Need join in addition to the eagerload since it is used in
+        #   the .count() query which doesn't respect the eagerload options  (could eliminate this with #5523).
+        # - Undefer average_rating column to prevent loading individual ratings per-history.
+        # - Eager load annotations - this causes a left join which might be inefficient if there were
+        #   potentially many items per history (like if joining HDAs for instance) but there should only
+        #   be at most one so this is fine.
+        return trans.sa_session.query(self.model_class).join("user").options(eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating"))
 
     def apply_query_filter(self, trans, query, **kwargs):
         # A public history is published, has a slug, and is not deleted.

lib/galaxy/webapps/galaxy/controllers/page.py

@@ -2,6 +2,7 @@ from json import loads
 
 from markupsafe import escape
 from sqlalchemy import and_, desc, false, true
+from sqlalchemy.orm import eagerload, undefer
 
 from galaxy import managers, model, util, web
 from galaxy.model.item_attrs import UsesItemRatings
@@ -82,8 +83,8 @@ class PageAllPublishedGrid(grids.Grid):
     )
 
     def build_initial_query(self, trans, **kwargs):
-        # Join so that searching history.user makes sense.
-        return trans.sa_session.query(self.model_class).join(model.User.table)
+        # See optimization description comments and TODO for tags in matching public histories query.
+        return trans.sa_session.query(self.model_class).join("user").options(eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating"))
 
     def apply_query_filter(self, trans, query, **kwargs):
         return query.filter(self.model_class.deleted == false()).filter(self.model_class.published == true())

lib/galaxy/webapps/galaxy/controllers/visualization.py

@@ -12,6 +12,7 @@ from paste.httpexceptions import (
 )
 from six import string_types
 from sqlalchemy import and_, desc, false, or_, true
+from sqlalchemy.orm import eagerload, undefer
 
 from galaxy import managers, model, util, web
 from galaxy.datatypes.interval import Bed
@@ -212,8 +213,8 @@ class VisualizationAllPublishedGrid(grids.Grid):
     )
 
     def build_initial_query(self, trans, **kwargs):
-        # Join so that searching history.user makes sense.
-        return trans.sa_session.query(self.model_class).join(model.User.table)
+        # See optimization description comments and TODO for tags in matching public histories query.
+        return trans.sa_session.query(self.model_class).join("user").options(eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating"))
 
     def apply_query_filter(self, trans, query, **kwargs):
         return query.filter(self.model_class.deleted == false()).filter(self.model_class.published == true())

lib/galaxy/webapps/galaxy/controllers/workflow.py

@@ -10,7 +10,7 @@ import requests
 from markupsafe import escape
 from six.moves.http_client import HTTPConnection
 from sqlalchemy import and_
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer
 from sqlalchemy.sql import expression
 
 from galaxy import (
@@ -138,8 +138,10 @@ class StoredWorkflowAllPublishedGrid(grids.Grid):
     ]
 
     def build_initial_query(self, trans, **kwargs):
-        # Join so that searching stored_workflow.user makes sense.
-        return trans.sa_session.query(self.model_class).join(model.User.table)
+        # See optimization description comments and TODO for tags in matching public histories query.
+        # In addition to that - be sure to lazyload the latest_workflow - it isn't needed and it causes all
+        # of its steps to be eagerly loaded.
+        return trans.sa_session.query(self.model_class).join("user").options(lazyload("latest_workflow"), eagerload("user").load_only("username"), eagerload("annotations"), undefer("average_rating"))
 
     def apply_query_filter(self, trans, query, **kwargs):
         # A public workflow is published, has a slug, and is not deleted.