Merge pull request #5993 from natefoo/cgroup-metric-enhancements

Minor enhancements for cgroup metrics

config/job_metrics_conf.xml.sample

@@ -38,9 +38,15 @@
   <!-- <env variables="HOSTNAME,SLURM_CPUS_ON_NODE,SLURM_JOBID" /> -->
 
   <!-- If galaxy jobs are run in cgroups, like slurm does if memory limits
-       are enforced, we can try to grep some information from this. -->
+       are enforced, we can try to grep some information from this. By default,
+       only a small set of cgroup parameters will be recorded, the list of which
+       can be found in lib/galaxy/jobs/metrics/instrumenters/cgroup.py.  -->
   <!-- <cgroup /> -->
+  <!-- All params from the `cpuacct,cpu` and `memory` controllers can be
+       recorded.  -->
   <!-- <cgroup verbose="true" /> -->
+  <!-- Or, specific params can be recorded. -->
+  <!-- <cgroup params="cpuacct.usage,memory.max_usage_in_bytes,memory.memsw.max_usage_in_bytes" /> -->
 
   <!-- Uncomment to record hostname - *nix only -->
   <!-- <hostname /> -->

lib/galaxy/jobs/metrics/instrumenters/cgroup.py

@@ -54,6 +54,12 @@ class CgroupPlugin(InstrumentPlugin):
 
     def __init__(self, **kwargs):
         self.verbose = asbool(kwargs.get("verbose", False))
+        params_str = kwargs.get("params", None)
+        if params_str:
+            params = [v.strip() for v in params_str.split(",")]
+        else:
+            params = TITLES.keys()
+        self.params = params
 
     def post_execute_instrument(self, job_directory):
         commands = []
@@ -92,7 +98,7 @@ class CgroupPlugin(InstrumentPlugin):
         return metrics
 
     def __add_metric(self, metrics, metric):
-        if metric and (metric.subkey in TITLES or self.verbose):
+        if metric and (metric.subkey in self.params or self.verbose):
             metrics[metric.subkey] = metric.value
 
     def __read_key_value(self, line, prev_metric):

lib/galaxy/model/__init__.py

@@ -66,9 +66,10 @@ _datatypes_registry = None
 # this be unlimited - filter in Python if over this limit.
 MAX_IN_FILTER_LENGTH = 100
 
-# The column sizes for job metrics
+# The column sizes for job metrics. Note: changing these values does not change the column sizes, a migration must be
+# performed to do that.
 JOB_METRIC_MAX_LENGTH = 1023
-JOB_METRIC_PRECISION = 22
+JOB_METRIC_PRECISION = 26
 JOB_METRIC_SCALE = 7
 
 

lib/galaxy/model/migrate/versions/0142_change_numeric_metric_precision.py

@@ -0,0 +1,30 @@
+"""
+Migration script to change the 'value' column of 'user_preference' table from numeric(22, 7) to numeric(26, 7)
+"""
+from __future__ import print_function
+
+import logging
+
+from sqlalchemy import MetaData, Numeric, Table
+
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+
+def upgrade(migrate_engine):
+    print(__doc__)
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    try:
+        t = Table("job_metric_numeric", metadata, autoload=True)
+        t.c.metric_value.alter(type=Numeric(26, 7))
+        t = Table("task_metric_numeric", metadata, autoload=True)
+        t.c.metric_value.alter(type=Numeric(26, 7))
+    except Exception as e:
+        log.exception("Modifying numeric column failed: %s", str(e))
+
+
+def downgrade(migrate_engine):
+    # truncating columns would require truncating data in those columns, so it's best not to downgrade them
+    pass