From 4ac6c9f2e96ad837539c7fd8c32f2ac104c48a3c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Apr 2022 11:28:29 +0200 Subject: [PATCH 1/5] fix profile of multi_output_recurse feature was merged in july 21 --- test/functional/tools/multi_output_recurse.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/tools/multi_output_recurse.xml b/test/functional/tools/multi_output_recurse.xml index e8be0f3e8e8..ad2024eed4c 100644 --- a/test/functional/tools/multi_output_recurse.xml +++ b/test/functional/tools/multi_output_recurse.xml @@ -1,4 +1,4 @@ - + '$report'; mkdir subdir1; From 72be35fa8ab65094097567ca04a7e0b32e520794 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Apr 2022 12:14:58 +0200 Subject: [PATCH 2/5] fix error message on collection order by using the correct variable --- lib/galaxy/tool_util/verify/interactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index b2eda787436..b2af1441e6e 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -994,7 +994,7 @@ def verify_collection(output_collection_def, data_collection, verify_dataset): try: i = generated_sort_order[i:].index(identifier) + 1 except ValueError: - message = f"Output collection '{name}': identifier '{element_identifier}' found out of order, expected order of {expected_sort_order} for the tool generated collection elements {eo_ids}" + message = f"Output collection '{name}': identifier '{identifier}' found out of order, expected order of {expected_sort_order} for the tool generated collection elements {eo_ids}" raise AssertionError(message) verify_elements(data_collection["elements"], output_collection_def.element_tests) From 52be5c9cc76d0cf3030e033e4c4b01629e057687 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Apr 2022 12:17:13 +0200 Subject: [PATCH 3/5] fix sorting for recursively discovered datasets files in subdirectories were not subject to sort --- lib/galaxy/job_execution/output_collect.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index d5fe7ac089a..e5b2f22af2f 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -603,10 +603,9 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director new_parent_paths = parent_paths[:] new_parent_paths.append(filename) # The current directory is already validated, so use that as the next job_working_directory when recursing - for match in walk_over_extra_files( + matches.extend(walk_over_extra_files( filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths - ): - yield match + )) else: match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths) if match: From 297f7e51ef54d3416f36cde16a610188ba3dae7a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Apr 2022 14:07:43 +0200 Subject: [PATCH 4/5] refactor: do not store intermediate lists --- lib/galaxy/job_execution/output_collect.py | 39 ++++++++++++---------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index e5b2f22af2f..ac35b639917 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -592,26 +592,29 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director match the given collector's match criteria. If the collector has the recurse flag enabled, will also recursively descend into child folders. """ - matches = [] parent_paths = parent_paths or [] - directory = discover_target_directory(target_dir, job_working_directory) - if os.path.isdir(directory): - for filename in os.listdir(directory): - path = os.path.join(directory, filename) - if os.path.isdir(path): - if extra_file_collector.recurse: - new_parent_paths = parent_paths[:] - new_parent_paths.append(filename) - # The current directory is already validated, so use that as the next job_working_directory when recursing - matches.extend(walk_over_extra_files( - filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths - )) - else: - match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths) - if match: - matches.append(match) - for match in extra_file_collector.sort(matches): + def _walk(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths): + directory = discover_target_directory(target_dir, job_working_directory) + if os.path.isdir(directory): + for filename in os.listdir(directory): + path = os.path.join(directory, filename) + if os.path.isdir(path): + if extra_file_collector.recurse: + new_parent_paths = parent_paths[:] + new_parent_paths.append(filename) + # The current directory is already validated, so use that as the next job_working_directory when recursing + yield from _walk( + filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths + ) + else: + match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths) + if match: + yield match + + for match in extra_file_collector.sort( + _walk(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths) + ): yield match From 0626e45b3c8306700412c00cea664a358f09f881 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Apr 2022 14:09:12 +0200 Subject: [PATCH 5/5] add test for correct sorting on recursive discovery --- .../tools/multi_output_recurse_collection.xml | 57 +++++++++++++++++++ test/functional/tools/samples_tool_conf.xml | 1 + 2 files changed, 58 insertions(+) create mode 100644 test/functional/tools/multi_output_recurse_collection.xml diff --git a/test/functional/tools/multi_output_recurse_collection.xml b/test/functional/tools/multi_output_recurse_collection.xml new file mode 100644 index 00000000000..bce5ec07362 --- /dev/null +++ b/test/functional/tools/multi_output_recurse_collection.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + 1/3.txt; + echo "4" > 1/4.txt; + mkdir 2; + echo "1" > 2/1.txt; + echo "2" > 2/2.txt; + mkdir 3; + echo "5" > 3/5.txt; + mkdir 3/1; + echo "0" > 3/1/0.txt; + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 92870be5a6e..cdcd9fa30fb 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -28,6 +28,7 @@ +