Make displaying a zero resubmission count an admin's call

Hiding the zero is right for almost everyone and wrong for anyone who wants to see the
metric on every job, so put it behind a core option: show_zero_resubmissions, default
false. Parsed with asbool, since XML attributes arrive as strings and "false" is truthy.

Getting it to the rendered metric needed one fix. Formatting looked up
plugin_classes[plugin].formatter -- the class attribute, which CorePlugin assigns exactly
once, so whichever core plugin happened to be constructed first fixed the formatter for
the whole process and no configured option could reach it. That is the same reason the
existing timezone option only ever honours one configuration.

Ask the configured plugin first, mirroring the safety lookup a few lines below, and keep
the class attribute as the fallback for metrics whose plugin is no longer configured. The
default instrumenter is the one consulted, because metrics render without knowing which
destination the job ran on.

doc/source/admin/job_metrics.rst

@@ -58,7 +58,16 @@ survives a resubmission clearing or recreating that directory.
 
 It is recorded for every job but only *displayed* when it is non-zero, since almost no job is
 ever resubmitted and a zero would otherwise appear on every job's metrics panel. The value is
-stored either way, so queries and reports see a uniform metric.
+stored either way, so queries and reports see a uniform metric. Set the optional
+``show_zero_resubmissions`` option (default: ``false``) to display it on every job:
+
+.. code-block:: yaml
+
+    - type: core
+      show_zero_resubmissions: true
+
+Display options are read from the default metrics configuration rather than a per-destination
+one, because metrics are rendered without reference to the destination the job ran on.
 
 This counts both configured ``resubmit`` rules and runner-triggered resubmissions such as
 Slurm node-failure recovery. It does not count scheduler-internal requeues that Galaxy never

lib/galaxy/job_metrics/__init__.py

@@ -95,13 +95,20 @@ class JobMetrics:
         """Find :class:`formatting.JobMetricFormatter` corresponding to instrumented plugin value.
 
         None means the plugin recorded this metric but does not want it displayed.
+
+        Asks the configured plugin first, the way the safety lookup below does, so that display
+        options an admin set travel from the metrics configuration to the rendered metric. The
+        default instrumenter is the one consulted: rendering happens without knowing which
+        destination the job ran on.
         """
-        if plugin in self.plugin_classes:
-            plugin_class = self.plugin_classes[plugin]
-            formatter = plugin_class.formatter
-        else:
+        formatter = None
+        configured_plugin = self.default_job_instrumenter.get_configured_plugin(plugin)
+        if configured_plugin is not None:
+            formatter = configured_plugin.formatter
+        if formatter is None and plugin in self.plugin_classes:
+            formatter = self.plugin_classes[plugin].formatter
+        if formatter is None:
             formatter = DEFAULT_FORMATTER
-        assert formatter
         return formatter.format(key, value)
 
     def dictifiable_metrics(self, raw_metrics: list[RawMetric], allowed_safety: Safety) -> list[DictifiableMetric]:

lib/galaxy/job_metrics/instrumenters/core.py

@@ -8,6 +8,8 @@ from typing import (
     Any,
 )
 
+from galaxy.util import asbool
+
 from . import (
     InstrumentPlugin,
     ProvidesJobMetricsContext,
@@ -32,9 +34,10 @@ RESUBMISSION_COUNT_KEY = "resubmission_count"
 
 
 class CorePluginFormatter(JobMetricFormatter):
-    def __init__(self, timezone: str | None):
+    def __init__(self, timezone: str | None, show_zero_resubmissions: bool = False):
         self.tz: zoneinfo.ZoneInfo | None = None
         self.strftime_format = "%Y-%m-%d %H:%M:%S"
+        self.show_zero_resubmissions = show_zero_resubmissions
         self.__init_tz(timezone)
 
     def __init_tz(self, timezone: str | None):
@@ -49,7 +52,7 @@ class CorePluginFormatter(JobMetricFormatter):
             return FormattedMetric("Container Type", value)
         value = int(value)
         if key == RESUBMISSION_COUNT_KEY:
-            if not value:
+            if not value and not self.show_zero_resubmissions:
                 # Recorded on every job so the metric means the same thing everywhere, but a
                 # count of zero describes almost every job and is not worth a row in the UI.
                 return None
@@ -76,11 +79,14 @@ class CorePlugin(InstrumentPlugin):
     default_safety = Safety.SAFE
 
     def __init__(self, **kwargs):
-        self.__init_formatter(kwargs.get("timezone"))
-
-    def __init_formatter(self, timezone: str | None):
+        self.formatter = CorePluginFormatter(
+            kwargs.get("timezone"),
+            show_zero_resubmissions=asbool(kwargs.get("show_zero_resubmissions", False)),
+        )
         if CorePlugin.formatter is None:
-            CorePlugin.formatter = CorePluginFormatter(timezone)
+            # Class-level fallback, for formatting metrics recorded by a plugin that is no
+            # longer in the metrics configuration and so has no instance to ask.
+            CorePlugin.formatter = self.formatter
 
     def pre_execute_instrument(self, job_directory: str) -> list[str]:
         commands = []

test/unit/job_metrics/test_core_resubmission.py

@@ -1,5 +1,6 @@
 """The core plugin's resubmission_count metric."""
 
+from galaxy.job_metrics import JobMetrics
 from galaxy.job_metrics.instrumenters.core import (
     CorePlugin,
     RESUBMISSION_COUNT_KEY,
@@ -36,6 +37,26 @@ def test_a_zero_count_is_recorded_but_not_displayed():
     assert plugin.formatter.format(RESUBMISSION_COUNT_KEY, 0) is None
 
 
+def test_show_zero_resubmissions_displays_the_zero():
+    plugin = CorePlugin(show_zero_resubmissions="true")
+
+    assert plugin.formatter is not None
+    assert plugin.formatter.format(RESUBMISSION_COUNT_KEY, 0) == ("Resubmission Count", "0")
+
+
+def test_show_zero_resubmissions_reaches_display_from_the_metrics_configuration():
+    """An XML attribute arrives as a string, and has to travel to the formatter that renders."""
+    job_metrics = JobMetrics(conf_dict=[{"type": "core", "show_zero_resubmissions": "true"}])
+
+    assert job_metrics.format("core", RESUBMISSION_COUNT_KEY, 0) == ("Resubmission Count", "0")
+
+
+def test_zero_resubmissions_stays_hidden_by_default_through_the_configuration():
+    job_metrics = JobMetrics(conf_dict=[{"type": "core"}])
+
+    assert job_metrics.format("core", RESUBMISSION_COUNT_KEY, 0) is None
+
+
 def test_resubmission_metric_formatting_and_safety():
     plugin = CorePlugin()