mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Implement collection.dataset_instances with query
Instead of recursively loading elements fro child collections.
We use this in a bunch of different places, but this makes the
check_inputs_ready check about twice as fast.
Before:
*** PROFILER RESULTS ***
check_inputs_ready (/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/model_operations.py:17)
function called 1 times
305330 function calls (299676 primitive calls) in 0.509 seconds
Ordered by: cumulative time, internal time, call count
List reduced from 717 to 40 due to restriction <40>
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.510 0.510 model_operations.py:17(check_inputs_ready)
1 0.000 0.000 0.507 0.507 __init__.py:246(_collect_inputs)
5314/4312 0.003 0.000 0.494 0.000 attributes.py:279(__get__)
1202/601 0.002 0.000 0.491 0.001 attributes.py:699(get)
601 0.003 0.000 0.482 0.001 strategies.py:665(_load_for_state)
501 0.001 0.000 0.473 0.001 <string>:1(<lambda>)
501 0.006 0.000 0.472 0.001 strategies.py:772(_emit_lazyload)
2 0.000 0.000 0.427 0.213 __init__.py:1513(visit_inputs)
2 0.000 0.000 0.427 0.213 __init__.py:21(visit_input_values)
4 0.000 0.000 0.427 0.107 __init__.py:117(callback_helper)
1 0.000 0.000 0.427 0.427 __init__.py:65(_collect_input_datasets)
2 0.000 0.000 0.427 0.213 __init__.py:81(visitor)
202/2 0.001 0.000 0.336 0.168 __init__.py:4116(dataset_instances)
501 0.005 0.000 0.243 0.000 baked.py:421(__iter__)
501 0.002 0.000 0.214 0.000 query.py:3501(_execute_and_instances)
200 0.005 0.000 0.204 0.001 baked.py:557(_load_on_pk_identity)
301 0.003 0.000 0.193 0.001 baked.py:539(all)
502 0.001 0.000 0.186 0.000 base.py:952(execute)
502 0.001 0.000 0.185 0.000 elements.py:296(_execute_on_connection)
502 0.003 0.000 0.184 0.000 base.py:1088(_execute_clauseelement)
502 0.004 0.000 0.175 0.000 base.py:1195(_execute_context)
502 0.000 0.000 0.152 0.000 default.py:589(do_execute)
200 0.000 0.000 0.152 0.001 __init__.py:4565(dataset_instance)
502 0.147 0.000 0.152 0.000 {method 'execute' of 'psycopg2.extensions.cursor' objects}
200 0.000 0.000 0.151 0.001 __init__.py:4554(element_object)
1001 0.005 0.000 0.144 0.000 loading.py:35(instances)
100 0.001 0.000 0.082 0.001 __init__.py:83(process_dataset)
101 0.000 0.000 0.080 0.001 __init__.py:155(auto_propagated_tags)
100 0.000 0.000 0.076 0.001 security.py:473(can_access_dataset)
100 0.000 0.000 0.076 0.001 security.py:1031(dataset_is_public)
501 0.001 0.000 0.060 0.000 loading.py:59(<listcomp>)
501 0.002 0.000 0.060 0.000 query.py:4345(row_processor)
502 0.001 0.000 0.059 0.000 result.py:1268(fetchall)
601/501 0.017 0.000 0.057 0.000 loading.py:354(_instance_processor)
502 0.001 0.000 0.054 0.000 result.py:926(_soft_close)
502 0.001 0.000 0.052 0.000 base.py:899(close)
502 0.001 0.000 0.051 0.000 base.py:1031(close)
502 0.001 0.000 0.050 0.000 base.py:858(_checkin)
502 0.002 0.000 0.050 0.000 base.py:671(_finalize_fairy)
501 0.002 0.000 0.042 0.000 baked.py:180(_add_lazyload_options)
after:
*** PROFILER RESULTS ***
check_inputs_ready (/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/model_operations.py:17)
function called 1 times
186085 function calls (183671 primitive calls) in 0.251 seconds
Ordered by: cumulative time, internal time, call count
List reduced from 738 to 40 due to restriction <40>
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.251 0.251 model_operations.py:17(check_inputs_ready)
2920/2718 0.002 0.000 0.200 0.000 attributes.py:279(__get__)
802/401 0.001 0.000 0.198 0.000 attributes.py:699(get)
401 0.002 0.000 0.192 0.000 strategies.py:665(_load_for_state)
301 0.001 0.000 0.187 0.001 <string>:1(<lambda>)
301 0.003 0.000 0.187 0.001 strategies.py:772(_emit_lazyload)
1 0.000 0.000 0.161 0.161 __init__.py:246(_collect_inputs)
201 0.002 0.000 0.111 0.001 baked.py:539(all)
301 0.002 0.000 0.107 0.000 baked.py:421(__iter__)
303 0.001 0.000 0.102 0.000 query.py:3501(_execute_and_instances)
2 0.000 0.000 0.099 0.049 __init__.py:1513(visit_inputs)
2 0.000 0.000 0.099 0.049 __init__.py:21(visit_input_values)
4 0.000 0.000 0.099 0.025 __init__.py:117(callback_helper)
1 0.000 0.000 0.099 0.099 __init__.py:65(_collect_input_datasets)
2 0.000 0.000 0.099 0.049 __init__.py:81(visitor)
1 0.000 0.000 0.091 0.091 __init__.py:2760(check_inputs_ready)
304 0.000 0.000 0.088 0.000 base.py:952(execute)
304 0.000 0.000 0.088 0.000 elements.py:296(_execute_on_connection)
304 0.002 0.000 0.087 0.000 base.py:1088(_execute_clauseelement)
304 0.002 0.000 0.078 0.000 base.py:1195(_execute_context)
101/1 0.000 0.000 0.072 0.072 __init__.py:4049(populated)
76 0.000 0.000 0.069 0.001 {built-in method builtins.all}
101 0.000 0.000 0.069 0.001 __init__.py:4053(<genexpr>)
100 0.001 0.000 0.069 0.001 __init__.py:83(process_dataset)
304 0.000 0.000 0.065 0.000 default.py:589(do_execute)
304 0.063 0.000 0.064 0.000 {method 'execute' of 'psycopg2.extensions.cursor' objects}
100 0.000 0.000 0.063 0.001 security.py:473(can_access_dataset)
100 0.000 0.000 0.063 0.001 security.py:1031(dataset_is_public)
803 0.003 0.000 0.062 0.000 loading.py:35(instances)
101 0.000 0.000 0.062 0.001 __init__.py:155(auto_propagated_tags)
100 0.002 0.000 0.053 0.001 baked.py:557(_load_on_pk_identity)
2 0.000 0.000 0.039 0.020 __init__.py:4116(dataset_instances)
2 0.000 0.000 0.032 0.016 query.py:3303(all)
304 0.001 0.000 0.029 0.000 result.py:1268(fetchall)
304 0.001 0.000 0.026 0.000 result.py:926(_soft_close)
304 0.001 0.000 0.025 0.000 base.py:899(close)
304 0.000 0.000 0.025 0.000 base.py:1031(close)
304 0.000 0.000 0.024 0.000 base.py:858(_checkin)
304 0.001 0.000 0.024 0.000 base.py:671(_finalize_fairy)
2 0.000 0.000 0.022 0.011 query.py:3476(__iter__)
This commit is contained in:
mvdbeek
@@ -4109,14 +4109,36 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
|
||||
|
||||
@property
|
||||
def dataset_instances(self):
|
||||
instances = []
|
||||
for element in self.elements:
|
||||
if element.is_collection:
|
||||
instances.extend(element.child_collection.dataset_instances)
|
||||
else:
|
||||
instance = element.dataset_instance
|
||||
instances.append(instance)
|
||||
return instances
|
||||
db_session = object_session(self)
|
||||
if db_session and self.id:
|
||||
dc = alias(DatasetCollection.table)
|
||||
de = alias(DatasetCollectionElement.table)
|
||||
hda = alias(HistoryDatasetAssociation.table)
|
||||
|
||||
depth_collection_type = self.collection_type
|
||||
select_from = dc.outerjoin(de, de.c.dataset_collection_id == dc.c.id)
|
||||
|
||||
while ":" in depth_collection_type:
|
||||
child_collection = alias(DatasetCollection.table)
|
||||
child_collection_element = alias(DatasetCollectionElement.table)
|
||||
select_from = select_from.outerjoin(child_collection, child_collection.c.id == de.c.child_collection_id)
|
||||
select_from = select_from.outerjoin(child_collection_element, child_collection_element.c.dataset_collection_id == child_collection.c.id)
|
||||
|
||||
de = child_collection_element
|
||||
depth_collection_type = depth_collection_type.split(":", 1)[1]
|
||||
select_from = select_from.outerjoin(hda, hda.c.id == de.c.hda_id)
|
||||
select_stmt = select([hda]).select_from(select_from).where(dc.c.id == self.id).distinct()
|
||||
return db_session.query(HistoryDatasetAssociation).select_entity_from(select_stmt).all()
|
||||
else:
|
||||
# Sessionless context
|
||||
instances = []
|
||||
for element in self.elements:
|
||||
if element.is_collection:
|
||||
instances.extend(element.child_collection.dataset_instances)
|
||||
else:
|
||||
instance = element.dataset_instance
|
||||
instances.append(instance)
|
||||
return instances
|
||||
|
||||
@property
|
||||
def dataset_elements(self):
|
||||
|
||||
Reference in New Issue
Block a user