opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13688 from bernt-matthias/test/collection_order

Browse Source 
Fix sorting of recursively discovered datasets
This commit is contained in:
John Chilton
2022-04-08 10:28:19 -04:00
committed by GitHub
parent 6f3bdd39d9 0626e45b3c
commit 637036ebd5
5 changed files with 81 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/job_execution/output_collect.py
+21 -19
View File
@@ -592,27 +592,29 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director
match the given collector's match criteria. If the collector has the
recurse flag enabled, will also recursively descend into child folders.
"""
matches = []
parent_paths = parent_paths or []
directory = discover_target_directory(target_dir, job_working_directory)
if os.path.isdir(directory):
for filename in os.listdir(directory):
path = os.path.join(directory, filename)
if os.path.isdir(path):
if extra_file_collector.recurse:
new_parent_paths = parent_paths[:]
new_parent_paths.append(filename)
# The current directory is already validated, so use that as the next job_working_directory when recursing
for match in walk_over_extra_files(
filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths
):
yield match
else:
match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths)
if match:
matches.append(match)
for match in extra_file_collector.sort(matches):
def _walk(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths):
directory = discover_target_directory(target_dir, job_working_directory)
if os.path.isdir(directory):
for filename in os.listdir(directory):
path = os.path.join(directory, filename)
if os.path.isdir(path):
if extra_file_collector.recurse:
new_parent_paths = parent_paths[:]
new_parent_paths.append(filename)
# The current directory is already validated, so use that as the next job_working_directory when recursing
yield from _walk(
filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths
)
else:
match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths)
if match:
yield match
for match in extra_file_collector.sort(
_walk(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths)
):
yield match
lib/galaxy/tool_util/verify/interactor.py
+1 -1
View File
@@ -994,7 +994,7 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
try:
i = generated_sort_order[i:].index(identifier) + 1
except ValueError:
message = f"Output collection '{name}': identifier '{element_identifier}' found out of order, expected order of {expected_sort_order} for the tool generated collection elements {eo_ids}"
message = f"Output collection '{name}': identifier '{identifier}' found out of order, expected order of {expected_sort_order} for the tool generated collection elements {eo_ids}"
raise AssertionError(message)
verify_elements(data_collection["elements"], output_collection_def.element_tests)
test/functional/tools/multi_output_recurse.xml
+1 -1
View File
@@ -1,4 +1,4 @@
<tool id="multi_output_recurse" name="multi_output_recurse" version="0.1.0" profile="18.01">
<tool id="multi_output_recurse" name="multi_output_recurse" version="0.1.0" profile="21.09">
<command><![CDATA[
echo "Hello" > '$report';
mkdir subdir1;
test/functional/tools/multi_output_recurse_collection.xml
+57
View File
@@ -0,0 +1,57 @@
<!--essentially the same as multi_output_recurse_collection, but:
- test collection and generated order in the collection
-->
<tool id="multi_output_recurse_collection" name="multi_output_recurse_collection" version="0.1.0" profile="22.01">
<macros>
<xml name="outelem" tokens="number" token_path="">
<element name="@PATH@@NUMBER@" ftype="txt">
<assert_contents>
<has_text text="@NUMBER@"/>
</assert_contents>
</element>
</xml>
</macros>
<command><![CDATA[
mkdir 1;
echo "3" > 1/3.txt;
echo "4" > 1/4.txt;
mkdir 2;
echo "1" > 2/1.txt;
echo "2" > 2/2.txt;
mkdir 3;
echo "5" > 3/5.txt;
mkdir 3/1;
echo "0" > 3/1/0.txt;
]]></command>
<inputs>
<param type="boolean" name="match_relative"/>
</inputs>
<outputs>
<collection type="list" name="output">
<discover_datasets directory="./" recurse="true" pattern="__designation_and_ext__"/>
</collection>
<collection type="list" name="output_relative">
<discover_datasets directory="./" recurse="true" pattern="__designation_and_ext__" match_relative_path="true"/>
</collection>
</outputs>
<tests>
<test>
<output_collection name="output" count="6">
<expand macro="outelem" number="0"/>
<expand macro="outelem" number="1"/>
<expand macro="outelem" number="2"/>
<expand macro="outelem" number="3"/>
<expand macro="outelem" number="4"/>
<expand macro="outelem" number="5"/>
</output_collection>
<output_collection name="output_relative" count="6">
<expand macro="outelem" path="1/" number="3"/>
<expand macro="outelem" path="1/" number="4"/>
<expand macro="outelem" path="2/" number="1"/>
<expand macro="outelem" path="2/" number="2"/>
<expand macro="outelem" path="3/" number="5"/>
<expand macro="outelem" path="3/1/" number="0"/>
</output_collection>
</test>
</tests>
</tool>
test/functional/tools/samples_tool_conf.xml
+1
View File
@@ -28,6 +28,7 @@
<tool file="multi_output_assign_primary.xml" />
<tool file="multi_output_assign_primary_ext_dbkey.xml" />
<tool file="multi_output_recurse.xml" />
<tool file="multi_output_recurse_collection.xml" />
<tool file="tool_provided_metadata_1.xml" />
<tool file="tool_provided_metadata_2.xml" />
<tool file="tool_provided_metadata_3.xml" />