Add total_walltime limit type

Add to sample config, config reader and unittest

config/job_conf.xml.sample_advanced

@@ -762,6 +762,13 @@
                 will be terminated by Galaxy.
          -->
         <limit type="walltime">24:00:00</limit>
+	<!-- total_walltime:
+		Total walltime that jobs may not exceed during during a
+		set period.  If total walltime of finished jobs exceeds
+		this value, any new jobs are paused.  window is a number
+		in days, representing the period.
+         -->
+        <limit type="total_walltime" window="30">24:00:00</limit>
         <!-- output_size:
                 Size that any defined tool output can grow to before the job
                 will be terminated. This does not include temporary files

lib/galaxy/jobs/__init__.py

@@ -265,12 +265,14 @@ class JobConfiguration( object ):
         types = dict(registered_user_concurrent_jobs=int,
                      anonymous_user_concurrent_jobs=int,
                      walltime=str,
+                     total_walltime=str,
                      output_size=util.size_to_bytes)
 
         self.limits = Bunch(registered_user_concurrent_jobs=None,
                             anonymous_user_concurrent_jobs=None,
                             walltime=None,
                             walltime_delta=None,
+                            total_walltime={},
                             output_size=None,
                             destination_user_concurrent_jobs={},
                             destination_total_concurrent_jobs={})
@@ -287,6 +289,13 @@ class JobConfiguration( object ):
                         self.limits.destination_total_concurrent_jobs[id] = int(limit.text)
                     else:
                         self.limits.destination_user_concurrent_jobs[id] = int(limit.text)
+                elif type == 'total_walltime':
+                    self.limits.total_walltime["window"] = (
+                        int(limit.get('window')) or 30
+                    )
+                    self.limits.total_walltime["raw"] = (
+                        types.get(type, str)(limit.text)
+                    )
                 elif limit.text:
                     self.limits.__dict__[type] = types.get(type, str)(limit.text)
 
@@ -294,6 +303,13 @@ class JobConfiguration( object ):
             h, m, s = [ int( v ) for v in self.limits.walltime.split( ':' ) ]
             self.limits.walltime_delta = datetime.timedelta( 0, s, 0, 0, m, h )
 
+        if "raw" in self.limits.total_walltime:
+            h, m, s = [ int( v ) for v in
+                        self.limits.total_walltime["raw"].split( ':' ) ]
+            self.limits.total_walltime["delta"] = datetime.timedelta(
+                0, s, 0, 0, m, h
+            )
+
         log.debug('Done loading job configuration')
 
     def __parse_job_conf_legacy(self):

test/unit/jobs/test_job_configuration.py

@@ -100,6 +100,7 @@ class JobConfXmlParserTestCase( unittest.TestCase ):
         assert limits.anonymous_user_concurrent_jobs is None
         assert limits.walltime is None
         assert limits.walltime_delta is None
+        assert limits.total_walltime == {}
         assert limits.output_size is None
         assert limits.destination_user_concurrent_jobs == {}
         assert limits.destination_total_concurrent_jobs == {}
@@ -113,6 +114,8 @@ class JobConfXmlParserTestCase( unittest.TestCase ):
         assert limits.destination_user_concurrent_jobs[ "mycluster" ] == 2
         assert limits.destination_user_concurrent_jobs[ "longjobs" ] == 1
         assert limits.walltime_delta == datetime.timedelta( 0, 0, 0, 0, 0, 24 )
+        assert limits.total_walltime["delta"] == datetime.timedelta( 0, 0, 0, 0, 0, 24)
+        assert limits.total_walltime["window"] == 30
 
     def test_env_parsing( self ):
         self.__with_advanced_config()