From 47a0d2c7c5a93ee2b7548fe068aa98bfd640140d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 26 Sep 2020 17:53:30 +0200 Subject: [PATCH] Implement collection.dataset_instances with query Instead of recursively loading elements fro child collections. We use this in a bunch of different places, but this makes the check_inputs_ready check about twice as fast. Before: *** PROFILER RESULTS *** check_inputs_ready (/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/model_operations.py:17) function called 1 times 305330 function calls (299676 primitive calls) in 0.509 seconds Ordered by: cumulative time, internal time, call count List reduced from 717 to 40 due to restriction <40> ncalls tottime percall cumtime percall filename:lineno(function) 1 0.000 0.000 0.510 0.510 model_operations.py:17(check_inputs_ready) 1 0.000 0.000 0.507 0.507 __init__.py:246(_collect_inputs) 5314/4312 0.003 0.000 0.494 0.000 attributes.py:279(__get__) 1202/601 0.002 0.000 0.491 0.001 attributes.py:699(get) 601 0.003 0.000 0.482 0.001 strategies.py:665(_load_for_state) 501 0.001 0.000 0.473 0.001 :1() 501 0.006 0.000 0.472 0.001 strategies.py:772(_emit_lazyload) 2 0.000 0.000 0.427 0.213 __init__.py:1513(visit_inputs) 2 0.000 0.000 0.427 0.213 __init__.py:21(visit_input_values) 4 0.000 0.000 0.427 0.107 __init__.py:117(callback_helper) 1 0.000 0.000 0.427 0.427 __init__.py:65(_collect_input_datasets) 2 0.000 0.000 0.427 0.213 __init__.py:81(visitor) 202/2 0.001 0.000 0.336 0.168 __init__.py:4116(dataset_instances) 501 0.005 0.000 0.243 0.000 baked.py:421(__iter__) 501 0.002 0.000 0.214 0.000 query.py:3501(_execute_and_instances) 200 0.005 0.000 0.204 0.001 baked.py:557(_load_on_pk_identity) 301 0.003 0.000 0.193 0.001 baked.py:539(all) 502 0.001 0.000 0.186 0.000 base.py:952(execute) 502 0.001 0.000 0.185 0.000 elements.py:296(_execute_on_connection) 502 0.003 0.000 0.184 0.000 base.py:1088(_execute_clauseelement) 502 0.004 0.000 0.175 0.000 base.py:1195(_execute_context) 502 0.000 0.000 0.152 0.000 default.py:589(do_execute) 200 0.000 0.000 0.152 0.001 __init__.py:4565(dataset_instance) 502 0.147 0.000 0.152 0.000 {method 'execute' of 'psycopg2.extensions.cursor' objects} 200 0.000 0.000 0.151 0.001 __init__.py:4554(element_object) 1001 0.005 0.000 0.144 0.000 loading.py:35(instances) 100 0.001 0.000 0.082 0.001 __init__.py:83(process_dataset) 101 0.000 0.000 0.080 0.001 __init__.py:155(auto_propagated_tags) 100 0.000 0.000 0.076 0.001 security.py:473(can_access_dataset) 100 0.000 0.000 0.076 0.001 security.py:1031(dataset_is_public) 501 0.001 0.000 0.060 0.000 loading.py:59() 501 0.002 0.000 0.060 0.000 query.py:4345(row_processor) 502 0.001 0.000 0.059 0.000 result.py:1268(fetchall) 601/501 0.017 0.000 0.057 0.000 loading.py:354(_instance_processor) 502 0.001 0.000 0.054 0.000 result.py:926(_soft_close) 502 0.001 0.000 0.052 0.000 base.py:899(close) 502 0.001 0.000 0.051 0.000 base.py:1031(close) 502 0.001 0.000 0.050 0.000 base.py:858(_checkin) 502 0.002 0.000 0.050 0.000 base.py:671(_finalize_fairy) 501 0.002 0.000 0.042 0.000 baked.py:180(_add_lazyload_options) after: *** PROFILER RESULTS *** check_inputs_ready (/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/model_operations.py:17) function called 1 times 186085 function calls (183671 primitive calls) in 0.251 seconds Ordered by: cumulative time, internal time, call count List reduced from 738 to 40 due to restriction <40> ncalls tottime percall cumtime percall filename:lineno(function) 1 0.000 0.000 0.251 0.251 model_operations.py:17(check_inputs_ready) 2920/2718 0.002 0.000 0.200 0.000 attributes.py:279(__get__) 802/401 0.001 0.000 0.198 0.000 attributes.py:699(get) 401 0.002 0.000 0.192 0.000 strategies.py:665(_load_for_state) 301 0.001 0.000 0.187 0.001 :1() 301 0.003 0.000 0.187 0.001 strategies.py:772(_emit_lazyload) 1 0.000 0.000 0.161 0.161 __init__.py:246(_collect_inputs) 201 0.002 0.000 0.111 0.001 baked.py:539(all) 301 0.002 0.000 0.107 0.000 baked.py:421(__iter__) 303 0.001 0.000 0.102 0.000 query.py:3501(_execute_and_instances) 2 0.000 0.000 0.099 0.049 __init__.py:1513(visit_inputs) 2 0.000 0.000 0.099 0.049 __init__.py:21(visit_input_values) 4 0.000 0.000 0.099 0.025 __init__.py:117(callback_helper) 1 0.000 0.000 0.099 0.099 __init__.py:65(_collect_input_datasets) 2 0.000 0.000 0.099 0.049 __init__.py:81(visitor) 1 0.000 0.000 0.091 0.091 __init__.py:2760(check_inputs_ready) 304 0.000 0.000 0.088 0.000 base.py:952(execute) 304 0.000 0.000 0.088 0.000 elements.py:296(_execute_on_connection) 304 0.002 0.000 0.087 0.000 base.py:1088(_execute_clauseelement) 304 0.002 0.000 0.078 0.000 base.py:1195(_execute_context) 101/1 0.000 0.000 0.072 0.072 __init__.py:4049(populated) 76 0.000 0.000 0.069 0.001 {built-in method builtins.all} 101 0.000 0.000 0.069 0.001 __init__.py:4053() 100 0.001 0.000 0.069 0.001 __init__.py:83(process_dataset) 304 0.000 0.000 0.065 0.000 default.py:589(do_execute) 304 0.063 0.000 0.064 0.000 {method 'execute' of 'psycopg2.extensions.cursor' objects} 100 0.000 0.000 0.063 0.001 security.py:473(can_access_dataset) 100 0.000 0.000 0.063 0.001 security.py:1031(dataset_is_public) 803 0.003 0.000 0.062 0.000 loading.py:35(instances) 101 0.000 0.000 0.062 0.001 __init__.py:155(auto_propagated_tags) 100 0.002 0.000 0.053 0.001 baked.py:557(_load_on_pk_identity) 2 0.000 0.000 0.039 0.020 __init__.py:4116(dataset_instances) 2 0.000 0.000 0.032 0.016 query.py:3303(all) 304 0.001 0.000 0.029 0.000 result.py:1268(fetchall) 304 0.001 0.000 0.026 0.000 result.py:926(_soft_close) 304 0.001 0.000 0.025 0.000 base.py:899(close) 304 0.000 0.000 0.025 0.000 base.py:1031(close) 304 0.000 0.000 0.024 0.000 base.py:858(_checkin) 304 0.001 0.000 0.024 0.000 base.py:671(_finalize_fairy) 2 0.000 0.000 0.022 0.011 query.py:3476(__iter__) --- lib/galaxy/model/__init__.py | 38 ++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6cf72fd1006..be537bed3df 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4109,14 +4109,36 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById): @property def dataset_instances(self): - instances = [] - for element in self.elements: - if element.is_collection: - instances.extend(element.child_collection.dataset_instances) - else: - instance = element.dataset_instance - instances.append(instance) - return instances + db_session = object_session(self) + if db_session and self.id: + dc = alias(DatasetCollection.table) + de = alias(DatasetCollectionElement.table) + hda = alias(HistoryDatasetAssociation.table) + + depth_collection_type = self.collection_type + select_from = dc.outerjoin(de, de.c.dataset_collection_id == dc.c.id) + + while ":" in depth_collection_type: + child_collection = alias(DatasetCollection.table) + child_collection_element = alias(DatasetCollectionElement.table) + select_from = select_from.outerjoin(child_collection, child_collection.c.id == de.c.child_collection_id) + select_from = select_from.outerjoin(child_collection_element, child_collection_element.c.dataset_collection_id == child_collection.c.id) + + de = child_collection_element + depth_collection_type = depth_collection_type.split(":", 1)[1] + select_from = select_from.outerjoin(hda, hda.c.id == de.c.hda_id) + select_stmt = select([hda]).select_from(select_from).where(dc.c.id == self.id).distinct() + return db_session.query(HistoryDatasetAssociation).select_entity_from(select_stmt).all() + else: + # Sessionless context + instances = [] + for element in self.elements: + if element.is_collection: + instances.extend(element.child_collection.dataset_instances) + else: + instance = element.dataset_instance + instances.append(instance) + return instances @property def dataset_elements(self):