Add the ability for tests to have a lines_diff attribute set for standard output datasets that will allow the specified number of lines to vary between test-data and tool run results; a 'change' to a line equates to a difference of 2 lines between files (i.e. 1 line added, 1 line removed). Add the ability for test comparisons to use single or multiline regular expressions, this is accessed by adding the attribute compare= "match_re" or "match_re_multiline" (respectively); the standard diff method can be explicitly declared as "diff", or is used at the default when not specified.

test/functional/test_toolbox.py

@@ -66,13 +66,13 @@ class ToolTestCase( TwillTestCase ):
         self.assertTrue( data_list )
         elem_index = 0 - len( testdef.outputs )
         for output_tuple in testdef.outputs:
-            name, file, sort = output_tuple
+            name, outfile, attributes = output_tuple
             # Get the correct hid
             elem = data_list[ elem_index ]
             self.assertTrue( elem is not None )
             elem_hid = elem.get( 'hid' )
             elem_index += 1
-            self.verify_dataset_correctness( file, hid=elem_hid, maxseconds=testdef.maxseconds, sort=sort )
+            self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes )
         self.delete_history( id=self.security.encode_id( latest_history.id ) )
 
     def __expand_grouping( self, tool_inputs, declared_inputs, prefix='' ):