Merge pull request #22153 from dannon/fix/binary-unit-labels

Use correct binary unit prefixes (KiB/MiB/GiB)

client/src/components/Masthead/QuotaMeter.test.ts

@@ -64,13 +64,13 @@ describe("QuotaMeter.vue", () => {
 
     it("shows total usage when there is no quota", async () => {
         {
-            const user = { ...FAKE_USER, total_disk_usage: 7168 };
+            const user = { ...FAKE_USER, total_disk_usage: 7000 };
             const config = { enable_quotas: false };
             const wrapper = await createQuotaMeterWrapper(config, user);
             expect(wrapper.find("span").text()).toBe("Using 7 KB");
         }
         {
-            const user = { ...FAKE_USER, total_disk_usage: 21504, quota: "unlimited" };
+            const user = { ...FAKE_USER, total_disk_usage: 21000, quota: "unlimited" };
             const config = { enable_quotas: true };
             const wrapper = await createQuotaMeterWrapper(config, user);
             expect(wrapper.find("span").text()).toBe("Using 21 KB");

client/src/components/User/DiskUsage/DiskUsageSummary.test.ts

@@ -20,16 +20,16 @@ const quotaUsageClassSelector = ".quota-usage";
 const basicDiskUsageSummaryId = "#basic-disk-usage-summary";
 
 const fakeUserWithQuota = getFakeRegisteredUser({
-    total_disk_usage: 1048576,
-    quota_bytes: 104857600,
+    total_disk_usage: 1000000,
+    quota_bytes: 100000000,
     quota_percent: 1,
 });
 
 const fakeQuotaUsages: UserQuotaUsageData[] = [
     {
         quota_source_label: "Default",
-        quota_bytes: 104857600,
-        total_disk_usage: 1048576,
+        quota_bytes: 100000000,
+        total_disk_usage: 1000000,
     },
 ];
 
@@ -92,8 +92,8 @@ describe("DiskUsageSummary.vue", () => {
         const updatedFakeQuotaUsages: UserQuotaUsageData[] = [
             {
                 quota_source_label: "Default",
-                quota_bytes: 104857600,
-                total_disk_usage: 2097152,
+                quota_bytes: 100000000,
+                total_disk_usage: 2000000,
             },
         ];
         server.use(

client/src/utils/utils.ts

@@ -197,7 +197,7 @@ export function roundToDecimalPlaces(number: number, numPlaces: number) {
     return parseFloat(number.toFixed(numPlaces));
 }
 
-const kb = 1024;
+const kb = 1000;
 const mb = kb * kb;
 const gb = mb * kb;
 const tb = gb * kb;

lib/galaxy/job_metrics/instrumenters/cgroup.py

@@ -68,7 +68,7 @@ TITLES = {
 CONVERSION = {
     "memory.oom_control.oom_kill_disable": lambda x: "No" if x == 1 else "Yes",
     "memory.oom_control.under_oom": lambda x: "Yes" if x == 1 else "No",
-    "memory.peak": lambda x: nice_size(x),
+    "memory.peak": lambda x: nice_size(x, binary=True),
     "cpuacct.usage": lambda x: formatting.seconds_to_str(x / 10**9),  # convert nanoseconds
     "cpu.stat.system_usec": lambda x: formatting.seconds_to_str(x / 10**6),  # convert microseconds
     "cpu.stat.usage_usec": lambda x: formatting.seconds_to_str(x / 10**6),  # convert microseconds
@@ -114,7 +114,7 @@ class CgroupPluginFormatter(formatting.JobMetricFormatter):
             return formatting.FormattedMetric(title, CONVERSION[key](value))
         elif key.endswith("_bytes"):
             try:
-                return formatting.FormattedMetric(title, nice_size(value))
+                return formatting.FormattedMetric(title, nice_size(value, binary=True))
             except ValueError:
                 pass
         else:

lib/galaxy/job_metrics/instrumenters/meminfo.py

@@ -17,7 +17,7 @@ MEMINFO_TITLES = {"memtotal": "Total System Memory", "swaptotal": "Total System
 class MemInfoFormatter(formatting.JobMetricFormatter):
     def format(self, key: str, value: Any) -> formatting.FormattedMetric:
         title = MEMINFO_TITLES.get(key, key)
-        return formatting.FormattedMetric(title, util.nice_size(value * 1000))  # kB = *1000, KB = *1024 - wikipedia
+        return formatting.FormattedMetric(title, util.nice_size(value * 1024, binary=True))
 
 
 class MemInfoPlugin(InstrumentPlugin):

lib/galaxy/jobs/runners/univa.py

@@ -277,7 +277,7 @@ class UnivaJobRunner(DRMAAJobRunner):
         # qdel       100     137          user@mail
 
         extinfo["time_wasted"] = _parse_time(qacct["wallclock"])
-        extinfo["memory_wasted"] = size_to_bytes(qacct["maxvmem"])
+        extinfo["memory_wasted"] = size_to_bytes(qacct["maxvmem"], binary=True)
         extinfo["slots"] = int(qacct["slots"])
 
         # deleted_by
@@ -608,7 +608,7 @@ def _parse_native_specs(job_id, native_spec):
     # parse memory
     m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec)
     if m is not None:
-        mem = size_to_bytes(m.group(1))
+        mem = size_to_bytes(m.group(1), binary=True)
         # mem = _parse_mem(m.group(1))
         if mem is None:
             log.error(f"DRMAAUniva: job {job_id} has unparsable memory native spec {native_spec}")

lib/galaxy/util/__init__.py

@@ -1552,31 +1552,42 @@ def shorten_with_metric_prefix(amount: int) -> str:
         return str(amount)
 
 
-def nice_size(size: Union[float, int, str, Decimal]) -> str:
+def nice_size(size: Union[float, int, str, Decimal], binary: bool = False) -> str:
     """
     Returns a readably formatted string with the size
 
     >>> nice_size(100)
     '100 bytes'
     >>> nice_size(10000)
-    '9.8 KB'
+    '10.0 KB'
     >>> nice_size(1000000)
-    '976.6 KB'
+    '1.0 MB'
     >>> nice_size(100000000)
-    '95.4 MB'
+    '100.0 MB'
+    >>> nice_size(1024, binary=True)
+    '1.0 KiB'
+    >>> nice_size(1048576, binary=True)
+    '1.0 MiB'
     """
     try:
         size = float(size)
     except ValueError:
         return "??? bytes"
-    size, prefix = metric_prefix(size, 1024)
-    if prefix == "":
-        return f"{int(size)} bytes"
+    if binary:
+        size, prefix = metric_prefix(size, 1024)
+        if prefix == "":
+            return f"{int(size)} bytes"
+        else:
+            return f"{size:.1f} {prefix}iB"
     else:
-        return f"{size:.1f} {prefix}B"
+        size, prefix = metric_prefix(size, 1000)
+        if prefix == "":
+            return f"{int(size)} bytes"
+        else:
+            return f"{size:.1f} {prefix}B"
 
 
-def size_to_bytes(size):
+def size_to_bytes(size, binary: bool = False):
     """
     Returns a number of bytes (as integer) if given a reasonably formatted string with the size
 
@@ -1587,16 +1598,21 @@ def size_to_bytes(size):
     >>> size_to_bytes('10 bytes')
     10
     >>> size_to_bytes('4k')
-    4096
+    4000
     >>> size_to_bytes('2.2 TB')
-    2418925581107
+    2200000000000
     >>> size_to_bytes('.01 TB')
-    10995116277
+    10000000000
     >>> size_to_bytes('1.b')
     1
     >>> size_to_bytes('1.2E2k')
-    122880
+    120000
+    >>> size_to_bytes('4k', binary=True)
+    4096
+    >>> size_to_bytes('1 MB', binary=True)
+    1048576
     """
+    base = 1024 if binary else 1000
     # The following number regexp is based on https://stackoverflow.com/questions/385558/extract-float-double-value/385597#385597
     size_re = re.compile(r"(?P<number>(\d+(\.\d*)?|\.\d+)(e[+-]?\d+)?)\s*(?P<multiple>[eptgmk]?(b|bytes?)?)?$")
     size_match = size_re.match(size.lower())
@@ -1607,17 +1623,17 @@ def size_to_bytes(size):
     if multiple == "" or multiple.startswith("b"):
         return int(number)
     elif multiple.startswith("k"):
-        return int(number * 1024)
+        return int(number * base)
     elif multiple.startswith("m"):
-        return int(number * 1024**2)
+        return int(number * base**2)
     elif multiple.startswith("g"):
-        return int(number * 1024**3)
+        return int(number * base**3)
     elif multiple.startswith("t"):
-        return int(number * 1024**4)
+        return int(number * base**4)
     elif multiple.startswith("p"):
-        return int(number * 1024**5)
+        return int(number * base**5)
     elif multiple.startswith("e"):
-        return int(number * 1024**6)
+        return int(number * base**6)
     else:
         raise ValueError(f"Unknown multiplier '{multiple}' in '{size}'")
 

test/unit/job_metrics/test_job_metrics.py

@@ -47,7 +47,7 @@ def test_job_metrics_format_cgroup():
         "memory.limit_in_bytes",
         9223372036854771712,
         assert_title="Memory limit on cgroup (MEM)",
-        assert_value="8.0 EB",
+        assert_value="8.0 EiB",
     )
     _assert_format(
         "cgroup",
@@ -61,7 +61,7 @@ def test_job_metrics_format_cgroup():
         "memory.peak",
         45097156608,
         assert_title="Max memory usage recorded",
-        assert_value="42.0 GB",
+        assert_value="42.0 GiB",
     )