Managers: remove UTC zone when parsing dates

This change creates a more complete regex for parsing dates used in filters. MySQL in particular errored when the UTC zone character 'Z' was appended to a valid iso 8601 date (because it's MySQL). The FilterParser.parse_date method now:
- strips both the separator 'T' and any ending 'Z'
- one space are allowed as the date/time separator
- milliseconds are permitted
- both time and time with milliseconds are optional
- any added timezone (besides the UTC Z) will cause an error
- feedback was made more clear
- tests were added (and should make the above more clear)

lib/galaxy/managers/base.py

@@ -930,6 +930,9 @@ class ModelFilterParser( HasAModelManager ):
         super( ModelFilterParser, self ).__init__( app, **kwargs )
         self.app = app
 
+        #: regex for testing/dicing iso8601 date strings, with optional time and ms, but allowing only UTC timezone
+        self.date_string_re = re.compile( r'^(\d{4}\-\d{2}\-\d{2})[T| ]{0,1}(\d{2}:\d{2}:\d{2}\.*\d*)*Z{0,1}$' )
+
         # dictionary containing parsing data for ORM/SQLAlchemy-based filters
         # ..note: although kind of a pain in the ass and verbose, opt-in/whitelisting allows more control
         #   over potentially expensive queries
@@ -1121,21 +1124,21 @@ class ModelFilterParser( HasAModelManager ):
 
     def parse_date( self, date_string ):
         """
-        Attempts to get an SQL-able(?) date string for a query filter.
+        Reformats a string containing either seconds from epoch or an iso8601 formated
+        date string into a new date string usable within a filter query.
+
+        Seconds from epoch can be a floating point value as well (i.e containing ms).
         """
-        # Attempts to parse epoch int back into date string
+        # assume it's epoch if no date separator is present
         try:
-            # MySQL may not support microseconds precision
-            epoch = round(float(date_string))
+            epoch = float( date_string )
             datetime_obj = datetime.datetime.fromtimestamp( epoch )
-            return datetime_obj.isoformat(' ')
+            return datetime_obj.isoformat( sep=' ' )
         except ValueError:
             pass
-        # Removes T from date string
-        if not hasattr( self, 'date_string_re' ):
-            self.date_string_re = re.compile( r'^(\d{4}\-\d{2}\-\d{2})T' )
-        date_string = self.date_string_re.sub(r'\1 ', date_string, count=1)
-        # MySQL may not support microseconds precision
-        if not hasattr( self, 'date_string_microseconds_re' ):
-            self.date_string_microseconds_re = re.compile(r'^(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})\.\d+')  # YYYY-MM-DD HH:MM:SS.mmmmmm
-        return self.date_string_microseconds_re.sub(r'\1', date_string, count=1)
+
+        match = self.date_string_re.match( date_string )
+        if match:
+            date_string = ' '.join([ group for group in match.groups() if group ])
+            return date_string
+        raise ValueError( 'datetime strings must be in the ISO 8601 format and in the UTC' )

test/unit/managers/test_HistoryContentsManager.py

@@ -5,6 +5,7 @@ import os
 import imp
 import unittest
 import random
+import datetime
 
 test_utils = imp.load_source( 'test_utils',
     os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) )
@@ -30,10 +31,10 @@ user4_data = dict( email='user4@user4.user4', username='user4', password=default
 
 
 # =============================================================================
-class HistoryAsContainerTestCase( BaseTestCase, CreatesCollectionsMixin ):
+class HistoryAsContainerBaseTestCase( BaseTestCase, CreatesCollectionsMixin ):
 
     def set_up_managers( self ):
-        super( HistoryAsContainerTestCase, self ).set_up_managers()
+        super( HistoryAsContainerBaseTestCase, self ).set_up_managers()
         self.history_manager = HistoryManager( self.app )
         self.hda_manager = hdas.HDAManager( self.app )
         self.collection_manager = collections.DatasetCollectionManager( self.app )
@@ -49,6 +50,10 @@ class HistoryAsContainerTestCase( BaseTestCase, CreatesCollectionsMixin ):
             element_identifiers=self.build_element_identifiers( hdas ) )
         return hdca
 
+
+# =============================================================================
+class HistoryAsContainerTestCase( HistoryAsContainerBaseTestCase ):
+
     def test_contents( self ):
         user2 = self.user_manager.create( **user2_data )
         history = self.history_manager.create( name='history', user=user2 )
@@ -303,6 +308,46 @@ class HistoryAsContainerTestCase( BaseTestCase, CreatesCollectionsMixin ):
         self.assertEqual( self.contents_manager.contents( history, filters=filters ), [ contents[1], contents[6] ])
 
 
+class HistoryContentsFilterParserTestCase( HistoryAsContainerBaseTestCase ):
+
+    def set_up_managers( self ):
+        super( HistoryContentsFilterParserTestCase, self ).set_up_managers()
+        self.filter_parser = history_contents.HistoryContentsFilters( self.app )
+
+    def test_date_parser( self ):
+        # -- seconds and milliseconds from epoch
+        self.log( 'should be able to parse epoch seconds' )
+        self.assertEqual( self.filter_parser.parse_date( '1234567890' ),
+            datetime.datetime.fromtimestamp( 1234567890 ).isoformat( sep=' ' ) )
+
+        self.log( 'should be able to parse floating point epoch seconds.milliseconds' )
+        self.assertEqual( self.filter_parser.parse_date( '1234567890.123' ),
+            datetime.datetime.fromtimestamp( 1234567890.123 ).isoformat( sep=' ' ) )
+
+        self.log( 'should error if bad epoch is used' )
+        self.assertRaises( ValueError, self.filter_parser.parse_date, '0x000234' )
+
+        # -- datetime strings
+        self.log( 'should allow date alone' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13' ), '2009-02-13' )
+
+        self.log( 'should allow date and time' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13 18:13:00' ), '2009-02-13 18:13:00' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13T18:13:00' ), '2009-02-13 18:13:00' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13T18:13:00Z' ), '2009-02-13 18:13:00' )
+
+        self.log( 'should allow date and time with milliseconds' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13 18:13:00.123' ), '2009-02-13 18:13:00.123' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13T18:13:00.123' ), '2009-02-13 18:13:00.123' )
+        self.assertEqual( self.filter_parser.parse_date( '2009-02-13T18:13:00.123Z' ), '2009-02-13 18:13:00.123' )
+
+        self.log( 'should error if timezone is added' )
+        self.assertRaises( ValueError, self.filter_parser.parse_date, '2009-02-13T18:13:00.123+0700' )
+
+        self.log( 'should error if locale is used' )
+        self.assertRaises( ValueError, self.filter_parser.parse_date, 'Fri Feb 13 18:31:30 2009' )
+
+
 # =============================================================================
 if __name__ == '__main__':
     # or more generally, nosetests test_resourcemanagers.py -s -v