Cgroupsv2 metrics instrumenter tests

test/unit/job_metrics/test_cgroups.py

@@ -1,6 +1,6 @@
 from galaxy.job_metrics.instrumenters.cgroup import CgroupPlugin
 
-CGROUP_PRODUCTION_EXAMPLE_2201 = """__cpu.cfs_period_us__
+CGROUPV1_PRODUCTION_EXAMPLE_2201 = """__cpu.cfs_period_us__
 100000
 __cpu.cfs_quota_us__
 -1
@@ -105,11 +105,156 @@ __memory.use_hierarchy__
 1
 """
 
+CGROUPV2_PRODUCTION_EXAMPLE_232 = """__cpu.idle__
+0
+__cpu.max__
+max 100000
+__cpu.max.burst__
+0
+__cpu.stat__
+usage_usec 8992210
+user_usec 6139150
+system_usec 2853059
+core_sched.force_idle_usec 0
+nr_periods 0
+nr_throttled 0
+throttled_usec 0
+nr_bursts 0
+burst_usec 0
+__cpu.weight__
+100
+__cpu.weight.nice__
+0
+__memory.current__
+139350016
+__memory.events__
+low 0
+high 0
+max 0
+oom 0
+oom_kill 0
+oom_group_kill 0
+__memory.events.local__
+low 0
+high 0
+max 0
+oom 0
+oom_kill 0
+oom_group_kill 0
+__memory.high__
+max
+__memory.low__
+0
+__memory.max__
+max
+__memory.min__
+0
+__memory.numa_stat__
+anon N0=864256
+file N0=129146880
+kernel_stack N0=32768
+pagetables N0=131072
+sec_pagetables N0=0
+shmem N0=0
+file_mapped N0=0
+file_dirty N0=0
+file_writeback N0=0
+swapcached N0=0
+anon_thp N0=0
+file_thp N0=0
+shmem_thp N0=0
+inactive_anon N0=819200
+active_anon N0=20480
+inactive_file N0=51507200
+active_file N0=77639680
+unevictable N0=0
+slab_reclaimable N0=8638552
+slab_unreclaimable N0=340136
+workingset_refault_anon N0=0
+workingset_refault_file N0=77
+workingset_activate_anon N0=0
+workingset_activate_file N0=0
+workingset_restore_anon N0=0
+workingset_restore_file N0=0
+workingset_nodereclaim N0=0
+__memory.oom.group__
+0
+__memory.peak__
+339906560
+__memory.reclaim__
+__memory.stat__
+anon 860160
+file 129146880
+kernel 9211904
+kernel_stack 32768
+pagetables 126976
+sec_pagetables 0
+percpu 0
+sock 0
+vmalloc 0
+shmem 0
+zswap 0
+zswapped 0
+file_mapped 0
+file_dirty 0
+file_writeback 0
+swapcached 0
+anon_thp 0
+file_thp 0
+shmem_thp 0
+inactive_anon 815104
+active_anon 20480
+inactive_file 51507200
+active_file 77639680
+unevictable 0
+slab_reclaimable 8642480
+slab_unreclaimable 340904
+slab 8983384
+workingset_refault_anon 0
+workingset_refault_file 77
+workingset_activate_anon 0
+workingset_activate_file 0
+workingset_restore_anon 0
+workingset_restore_file 0
+workingset_nodereclaim 0
+pgscan 0
+pgsteal 0
+pgscan_kswapd 0
+pgscan_direct 0
+pgsteal_kswapd 0
+pgsteal_direct 0
+pgfault 132306
+pgmajfault 524
+pgrefill 0
+pgactivate 18958
+pgdeactivate 0
+pglazyfree 0
+pglazyfreed 0
+zswpin 0
+zswpout 0
+thp_fault_alloc 19
+thp_collapse_alloc 0
+__memory.swap.current__
+0
+__memory.swap.events__
+high 0
+max 0
+fail 0
+__memory.swap.high__
+max
+__memory.swap.max__
+max
+__memory.zswap.current__
+0
+__memory.zswap.max__
+max
+"""
 
-def test_cgroup_collection(tmpdir):
+
+def test_cgroupv1_collection(tmpdir):
     plugin = CgroupPlugin()
     job_dir = tmpdir.mkdir("job")
-    job_dir.join("__instrument_cgroup__metrics").write(CGROUP_PRODUCTION_EXAMPLE_2201)
+    job_dir.join("__instrument_cgroup__metrics").write(CGROUPV1_PRODUCTION_EXAMPLE_2201)
     properties = plugin.job_properties(1, job_dir)
     assert "cpuacct.usage" in properties
     assert properties["cpuacct.usage"] == 7265342042
@@ -117,6 +262,17 @@ def test_cgroup_collection(tmpdir):
     assert properties["memory.limit_in_bytes"] == 9223372036854771712
 
 
+def test_cgroupv2_collection(tmpdir):
+    plugin = CgroupPlugin()
+    job_dir = tmpdir.mkdir("job")
+    job_dir.join("__instrument_cgroup__metrics").write(CGROUPV2_PRODUCTION_EXAMPLE_232)
+    properties = plugin.job_properties(1, job_dir)
+    assert "cpu.stat.usage_usec" in properties
+    assert properties["cpu.stat.usage_usec"] == 8992210
+    assert "memory.peak" in properties
+    assert properties["memory.peak"] == 339906560
+
+
 def test_instrumentation(tmpdir):
     # don't actually run the instrumentation but at least exercise the code the and make
     # sure templating includes cgroup_mount override.

test/unit/job_metrics/test_job_metrics.py

@@ -49,6 +49,20 @@ def test_job_metrics_format_cgroup():
         assert_title="Memory limit on cgroup (MEM)",
         assert_value="8.0 EB",
     )
+    _assert_format(
+        "cgroup",
+        "cpu.stat.usage_usec",
+        7982357892.000000,
+        assert_title="CPU usage time",
+        assert_value="2.0 hours and 13.0 minutes",
+    )
+    _assert_format(
+        "cgroup",
+        "memory.peak",
+        45097156608,
+        assert_title="Max memory usage recorded",
+        assert_value="42.0 GB",
+    )
 
 
 def test_job_metrics_uname():