diff --git a/doc/source/admin/job_metrics.rst b/doc/source/admin/job_metrics.rst index b0ac49cd774..a9e8c33cb62 100644 --- a/doc/source/admin/job_metrics.rst +++ b/doc/source/admin/job_metrics.rst @@ -58,7 +58,16 @@ survives a resubmission clearing or recreating that directory. It is recorded for every job but only *displayed* when it is non-zero, since almost no job is ever resubmitted and a zero would otherwise appear on every job's metrics panel. The value is -stored either way, so queries and reports see a uniform metric. +stored either way, so queries and reports see a uniform metric. Set the optional +``show_zero_resubmissions`` option (default: ``false``) to display it on every job: + +.. code-block:: yaml + + - type: core + show_zero_resubmissions: true + +Display options are read from the default metrics configuration rather than a per-destination +one, because metrics are rendered without reference to the destination the job ran on. This counts both configured ``resubmit`` rules and runner-triggered resubmissions such as Slurm node-failure recovery. It does not count scheduler-internal requeues that Galaxy never diff --git a/lib/galaxy/job_metrics/__init__.py b/lib/galaxy/job_metrics/__init__.py index f3841390d10..d1f5b07f2d1 100644 --- a/lib/galaxy/job_metrics/__init__.py +++ b/lib/galaxy/job_metrics/__init__.py @@ -95,13 +95,20 @@ class JobMetrics: """Find :class:`formatting.JobMetricFormatter` corresponding to instrumented plugin value. None means the plugin recorded this metric but does not want it displayed. + + Asks the configured plugin first, the way the safety lookup below does, so that display + options an admin set travel from the metrics configuration to the rendered metric. The + default instrumenter is the one consulted: rendering happens without knowing which + destination the job ran on. """ - if plugin in self.plugin_classes: - plugin_class = self.plugin_classes[plugin] - formatter = plugin_class.formatter - else: + formatter = None + configured_plugin = self.default_job_instrumenter.get_configured_plugin(plugin) + if configured_plugin is not None: + formatter = configured_plugin.formatter + if formatter is None and plugin in self.plugin_classes: + formatter = self.plugin_classes[plugin].formatter + if formatter is None: formatter = DEFAULT_FORMATTER - assert formatter return formatter.format(key, value) def dictifiable_metrics(self, raw_metrics: list[RawMetric], allowed_safety: Safety) -> list[DictifiableMetric]: diff --git a/lib/galaxy/job_metrics/instrumenters/core.py b/lib/galaxy/job_metrics/instrumenters/core.py index 2aa029c1a33..69714f3a590 100644 --- a/lib/galaxy/job_metrics/instrumenters/core.py +++ b/lib/galaxy/job_metrics/instrumenters/core.py @@ -8,6 +8,8 @@ from typing import ( Any, ) +from galaxy.util import asbool + from . import ( InstrumentPlugin, ProvidesJobMetricsContext, @@ -32,9 +34,10 @@ RESUBMISSION_COUNT_KEY = "resubmission_count" class CorePluginFormatter(JobMetricFormatter): - def __init__(self, timezone: str | None): + def __init__(self, timezone: str | None, show_zero_resubmissions: bool = False): self.tz: zoneinfo.ZoneInfo | None = None self.strftime_format = "%Y-%m-%d %H:%M:%S" + self.show_zero_resubmissions = show_zero_resubmissions self.__init_tz(timezone) def __init_tz(self, timezone: str | None): @@ -49,7 +52,7 @@ class CorePluginFormatter(JobMetricFormatter): return FormattedMetric("Container Type", value) value = int(value) if key == RESUBMISSION_COUNT_KEY: - if not value: + if not value and not self.show_zero_resubmissions: # Recorded on every job so the metric means the same thing everywhere, but a # count of zero describes almost every job and is not worth a row in the UI. return None @@ -76,11 +79,14 @@ class CorePlugin(InstrumentPlugin): default_safety = Safety.SAFE def __init__(self, **kwargs): - self.__init_formatter(kwargs.get("timezone")) - - def __init_formatter(self, timezone: str | None): + self.formatter = CorePluginFormatter( + kwargs.get("timezone"), + show_zero_resubmissions=asbool(kwargs.get("show_zero_resubmissions", False)), + ) if CorePlugin.formatter is None: - CorePlugin.formatter = CorePluginFormatter(timezone) + # Class-level fallback, for formatting metrics recorded by a plugin that is no + # longer in the metrics configuration and so has no instance to ask. + CorePlugin.formatter = self.formatter def pre_execute_instrument(self, job_directory: str) -> list[str]: commands = [] diff --git a/test/unit/job_metrics/test_core_resubmission.py b/test/unit/job_metrics/test_core_resubmission.py index c1af91216be..5c2959f1bae 100644 --- a/test/unit/job_metrics/test_core_resubmission.py +++ b/test/unit/job_metrics/test_core_resubmission.py @@ -1,5 +1,6 @@ """The core plugin's resubmission_count metric.""" +from galaxy.job_metrics import JobMetrics from galaxy.job_metrics.instrumenters.core import ( CorePlugin, RESUBMISSION_COUNT_KEY, @@ -36,6 +37,26 @@ def test_a_zero_count_is_recorded_but_not_displayed(): assert plugin.formatter.format(RESUBMISSION_COUNT_KEY, 0) is None +def test_show_zero_resubmissions_displays_the_zero(): + plugin = CorePlugin(show_zero_resubmissions="true") + + assert plugin.formatter is not None + assert plugin.formatter.format(RESUBMISSION_COUNT_KEY, 0) == ("Resubmission Count", "0") + + +def test_show_zero_resubmissions_reaches_display_from_the_metrics_configuration(): + """An XML attribute arrives as a string, and has to travel to the formatter that renders.""" + job_metrics = JobMetrics(conf_dict=[{"type": "core", "show_zero_resubmissions": "true"}]) + + assert job_metrics.format("core", RESUBMISSION_COUNT_KEY, 0) == ("Resubmission Count", "0") + + +def test_zero_resubmissions_stays_hidden_by_default_through_the_configuration(): + job_metrics = JobMetrics(conf_dict=[{"type": "core"}]) + + assert job_metrics.format("core", RESUBMISSION_COUNT_KEY, 0) is None + + def test_resubmission_metric_formatting_and_safety(): plugin = CorePlugin()