From 2cdb78d2cd8f2f0d90d823da4e185d3c0e79de72 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 18 Mar 2026 13:09:43 -0400 Subject: [PATCH] Add binary mode to nice_size/size_to_bytes for memory contexts We can keep or drop this, but this adapts nice_size and size_to_bytes with a binary=True option that uses 1024-based math and KiB/MiB/GiB labels. Wired it up in the memory-related callsites (cgroup, meminfo, univa runner) where binary units are more appropriate than SI. --- .../job_metrics/instrumenters/cgroup.py | 4 +- .../job_metrics/instrumenters/meminfo.py | 2 +- lib/galaxy/jobs/runners/univa.py | 4 +- lib/galaxy/util/__init__.py | 40 +++++++++++++------ test/unit/job_metrics/test_job_metrics.py | 4 +- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/job_metrics/instrumenters/cgroup.py b/lib/galaxy/job_metrics/instrumenters/cgroup.py index 4f1796c9da8..a391686a6ba 100644 --- a/lib/galaxy/job_metrics/instrumenters/cgroup.py +++ b/lib/galaxy/job_metrics/instrumenters/cgroup.py @@ -68,7 +68,7 @@ TITLES = { CONVERSION = { "memory.oom_control.oom_kill_disable": lambda x: "No" if x == 1 else "Yes", "memory.oom_control.under_oom": lambda x: "Yes" if x == 1 else "No", - "memory.peak": lambda x: nice_size(x), + "memory.peak": lambda x: nice_size(x, binary=True), "cpuacct.usage": lambda x: formatting.seconds_to_str(x / 10**9), # convert nanoseconds "cpu.stat.system_usec": lambda x: formatting.seconds_to_str(x / 10**6), # convert microseconds "cpu.stat.usage_usec": lambda x: formatting.seconds_to_str(x / 10**6), # convert microseconds @@ -114,7 +114,7 @@ class CgroupPluginFormatter(formatting.JobMetricFormatter): return formatting.FormattedMetric(title, CONVERSION[key](value)) elif key.endswith("_bytes"): try: - return formatting.FormattedMetric(title, nice_size(value)) + return formatting.FormattedMetric(title, nice_size(value, binary=True)) except ValueError: pass else: diff --git a/lib/galaxy/job_metrics/instrumenters/meminfo.py b/lib/galaxy/job_metrics/instrumenters/meminfo.py index 9e1d9be1de2..c9d6998f008 100644 --- a/lib/galaxy/job_metrics/instrumenters/meminfo.py +++ b/lib/galaxy/job_metrics/instrumenters/meminfo.py @@ -17,7 +17,7 @@ MEMINFO_TITLES = {"memtotal": "Total System Memory", "swaptotal": "Total System class MemInfoFormatter(formatting.JobMetricFormatter): def format(self, key: str, value: Any) -> formatting.FormattedMetric: title = MEMINFO_TITLES.get(key, key) - return formatting.FormattedMetric(title, util.nice_size(value * 1000)) # kB = *1000, KB = *1024 - wikipedia + return formatting.FormattedMetric(title, util.nice_size(value * 1024, binary=True)) class MemInfoPlugin(InstrumentPlugin): diff --git a/lib/galaxy/jobs/runners/univa.py b/lib/galaxy/jobs/runners/univa.py index 920083266c3..54b7c4556fc 100644 --- a/lib/galaxy/jobs/runners/univa.py +++ b/lib/galaxy/jobs/runners/univa.py @@ -277,7 +277,7 @@ class UnivaJobRunner(DRMAAJobRunner): # qdel 100 137 user@mail extinfo["time_wasted"] = _parse_time(qacct["wallclock"]) - extinfo["memory_wasted"] = size_to_bytes(qacct["maxvmem"]) + extinfo["memory_wasted"] = size_to_bytes(qacct["maxvmem"], binary=True) extinfo["slots"] = int(qacct["slots"]) # deleted_by @@ -608,7 +608,7 @@ def _parse_native_specs(job_id, native_spec): # parse memory m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec) if m is not None: - mem = size_to_bytes(m.group(1)) + mem = size_to_bytes(m.group(1), binary=True) # mem = _parse_mem(m.group(1)) if mem is None: log.error(f"DRMAAUniva: job {job_id} has unparsable memory native spec {native_spec}") diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index e936a1cbcfb..a8d6d37de61 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1551,7 +1551,7 @@ def shorten_with_metric_prefix(amount: int) -> str: return str(amount) -def nice_size(size: Union[float, int, str, Decimal]) -> str: +def nice_size(size: Union[float, int, str, Decimal], binary: bool = False) -> str: """ Returns a readably formatted string with the size @@ -1563,19 +1563,30 @@ def nice_size(size: Union[float, int, str, Decimal]) -> str: '1.0 MB' >>> nice_size(100000000) '100.0 MB' + >>> nice_size(1024, binary=True) + '1.0 KiB' + >>> nice_size(1048576, binary=True) + '1.0 MiB' """ try: size = float(size) except ValueError: return "??? bytes" - size, prefix = metric_prefix(size, 1000) - if prefix == "": - return f"{int(size)} bytes" + if binary: + size, prefix = metric_prefix(size, 1024) + if prefix == "": + return f"{int(size)} bytes" + else: + return f"{size:.1f} {prefix}iB" else: - return f"{size:.1f} {prefix}B" + size, prefix = metric_prefix(size, 1000) + if prefix == "": + return f"{int(size)} bytes" + else: + return f"{size:.1f} {prefix}B" -def size_to_bytes(size): +def size_to_bytes(size, binary: bool = False): """ Returns a number of bytes (as integer) if given a reasonably formatted string with the size @@ -1595,7 +1606,12 @@ def size_to_bytes(size): 1 >>> size_to_bytes('1.2E2k') 120000 + >>> size_to_bytes('4k', binary=True) + 4096 + >>> size_to_bytes('1 MB', binary=True) + 1048576 """ + base = 1024 if binary else 1000 # The following number regexp is based on https://stackoverflow.com/questions/385558/extract-float-double-value/385597#385597 size_re = re.compile(r"(?P(\d+(\.\d*)?|\.\d+)(e[+-]?\d+)?)\s*(?P[eptgmk]?(b|bytes?)?)?$") size_match = size_re.match(size.lower()) @@ -1606,17 +1622,17 @@ def size_to_bytes(size): if multiple == "" or multiple.startswith("b"): return int(number) elif multiple.startswith("k"): - return int(number * 1000) + return int(number * base) elif multiple.startswith("m"): - return int(number * 1000**2) + return int(number * base**2) elif multiple.startswith("g"): - return int(number * 1000**3) + return int(number * base**3) elif multiple.startswith("t"): - return int(number * 1000**4) + return int(number * base**4) elif multiple.startswith("p"): - return int(number * 1000**5) + return int(number * base**5) elif multiple.startswith("e"): - return int(number * 1000**6) + return int(number * base**6) else: raise ValueError(f"Unknown multiplier '{multiple}' in '{size}'") diff --git a/test/unit/job_metrics/test_job_metrics.py b/test/unit/job_metrics/test_job_metrics.py index a80df5e9b41..bb9acb053b8 100644 --- a/test/unit/job_metrics/test_job_metrics.py +++ b/test/unit/job_metrics/test_job_metrics.py @@ -47,7 +47,7 @@ def test_job_metrics_format_cgroup(): "memory.limit_in_bytes", 9223372036854771712, assert_title="Memory limit on cgroup (MEM)", - assert_value="8.0 EB", + assert_value="8.0 EiB", ) _assert_format( "cgroup", @@ -61,7 +61,7 @@ def test_job_metrics_format_cgroup(): "memory.peak", 45097156608, assert_title="Max memory usage recorded", - assert_value="42.0 GB", + assert_value="42.0 GiB", )