fix(kms): restore persisted configuration after restart (#6821)

* fix(kms): restore persisted configuration after restart

* docs(kms): cover the reload route and startup load states

The admin contract matrix pins every dynamic KMS route for the rc and
console handoff, so the new POST /kms/reload needs a row there, and the
reload response reuses the configure snapshot shape rather than adding a
wire type. The observability runbook gains the operator procedure the
reload exists for: telling a load_failed startup apart from a server
that was never configured, and recovering without resubmitting secrets.

crates/e2e_test/src/kms/configured_roundtrip_test.rs

@@ -17,6 +17,7 @@
 
 use super::common::{
     LocalKMSTestEnvironment, VAULT_KEY_NAME, VaultTestEnvironment, configure_kms, get_kms_status, kms_admin_request, start_kms,
+    test_sse_kms_encryption,
 };
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
@@ -431,6 +432,38 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
     Ok(())
 }
 
+#[tokio::test]
+async fn test_admin_configured_local_kms_is_restored_after_restart() -> TestResult {
+    let mut env = LocalKMSTestEnvironment::new().await?;
+    env.base_env.start_rustfs_server(Vec::new()).await?;
+
+    let default_key_id = env.configure_local_kms().await?;
+    start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
+
+    env.base_env.restart_server_preserving_data(Vec::new(), &[]).await?;
+    assert_configured_status(
+        &env.base_env.url,
+        &env.base_env.access_key,
+        &env.base_env.secret_key,
+        "local",
+        &default_key_id,
+    )
+    .await?;
+
+    let bucket = format!("kms-restart-{}", Uuid::new_v4());
+    env.base_env.create_test_bucket(&bucket).await?;
+    let client = env.base_env.create_s3_client();
+    test_sse_kms_encryption(&client, &bucket).await?;
+    client
+        .delete_object()
+        .bucket(&bucket)
+        .key("test-sse-kms-object")
+        .send()
+        .await?;
+    env.base_env.delete_test_bucket(&bucket).await?;
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
     let mut env = VaultTestEnvironment::new().await?;

crates/kms/src/service_manager.rs

@@ -259,6 +259,25 @@ impl KmsServiceManager {
         (state.status.clone(), config)
     }
 
+    /// Publish an initialization failure when no usable KMS state exists yet.
+    ///
+    /// Startup configuration discovery happens outside this crate. Recording
+    /// its failure here keeps status truthful without allowing a late failure
+    /// to replace an already configured or running service.
+    pub async fn record_initialization_error(&self, message: impl Into<String>) {
+        let _guard = self.lifecycle_mutex.lock().await;
+        let current = self.state.load_full();
+        if current.config.is_some() || current.current_service.is_some() {
+            return;
+        }
+
+        self.state.store(Arc::new(RuntimeState {
+            config: None,
+            status: KmsServiceStatus::Error(message.into()),
+            current_service: None,
+        }));
+    }
+
     fn redact_config(config: &mut KmsConfig) {
         if let BackendConfig::Static(static_config) = &mut config.backend_config {
             use zeroize::Zeroize;
@@ -849,6 +868,39 @@ mod tests {
         assert!(manager.get_encryption_service().await.is_none());
     }
 
+    #[tokio::test]
+    async fn initialization_error_is_visible_until_configuration_succeeds() {
+        let manager = KmsServiceManager::new();
+
+        manager
+            .record_initialization_error("persisted configuration could not be loaded")
+            .await;
+
+        assert_eq!(
+            manager.get_status().await,
+            KmsServiceStatus::Error("persisted configuration could not be loaded".to_string())
+        );
+        assert!(manager.get_config().await.is_none());
+
+        manager
+            .configure(static_config("key-a", 0x11))
+            .await
+            .expect("configure after startup failure");
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Configured);
+    }
+
+    #[tokio::test]
+    async fn initialization_error_never_replaces_a_running_service() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+
+        manager.record_initialization_error("late startup failure").await;
+
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert!(manager.get_encryption_service().await.is_some());
+    }
+
     #[tokio::test]
     async fn configure_rejects_running_service_without_changing_snapshot() {
         let manager = KmsServiceManager::new();

docs/operations/kms-admin-contract.md

@@ -10,6 +10,7 @@ The wire prefix is `/rustfs/admin/v3`. Request and response field names for the
 | `POST /kms/reconfigure` | `kms:Configure` / high | no | supported | supported | none |
 | `POST /kms/start` | `kms:ServiceControl` / high | no | supported | supported | none |
 | `POST /kms/stop` | `kms:ServiceControl` / high | no | supported | supported | none |
+| `POST /kms/reload` | `kms:ServiceControl` / high | no | pending | pending | Re-reads the cluster-persisted configuration without resubmitting secrets; response reuses the configure shape. |
 | `GET /kms/config` | `kms:Configure` / sensitive | no | no | supported | Redact operational paths before display. |
 | `POST /kms/clear-cache` | `kms:ClearCache` / high | no | no | supported | Keep the current `{status,message}` response stable. |
 | `POST /kms/keys` | `kms:Configure` / high | no | supported | supported | none |
@@ -52,7 +53,7 @@ One case is deliberately an error rather than a report: a listing that covered t
 
 ## Server-side snapshot coverage
 
-The merged #5626 producer snapshots cover the nine modern/legacy key response types and the metadata response type served by `kms_keys.rs` and `kms_key_metadata.rs`: create, describe, list, generate-data-key, delete, cancel-deletion, update-description, tag, and untag. The four dynamic responses served verbatim by `kms_dynamic.rs` are covered in `crates/kms/src/snapshots/`: configure, start, stop, and the `service-status` response.
+The merged #5626 producer snapshots cover the nine modern/legacy key response types and the metadata response type served by `kms_keys.rs` and `kms_key_metadata.rs`: create, describe, list, generate-data-key, delete, cancel-deletion, update-description, tag, and untag. The four dynamic responses served verbatim by `kms_dynamic.rs` are covered in `crates/kms/src/snapshots/`: configure, start, stop, and the `service-status` response. `POST /kms/reload` serves the same `ConfigureKmsResponse` type the configure snapshot pins; it adds no new wire shape.
 
 `POST /kms/clear-cache` now has a named `KmsClearCacheResponse` and a producer snapshot beside the others; its serialized bytes are unchanged from the inline JSON it replaced.
 
@@ -60,7 +61,7 @@ The remaining wire-shape gaps are intentionally documented rather than duplicate
 
 ## Client handoff gaps
 
-The `rc` client currently has status, key list/status/create/delete/cancel-deletion, configure/reconfigure/start/restart/stop, and diagnostic/roundtrip entry points. It has no lifecycle enable/disable/rotate, key metadata, or backup/restore commands. The console currently calls service-status, configure/reconfigure/start/stop/config, clear-cache, status, and the modern key CRUD routes. It has no lifecycle, metadata, or backup/restore UI. These pending cells are delivery items for `rustfs/cli` and `rustfs/console`; they are not implemented in this repository. A read-only issue search on 2026-08-02 found no matching KMS issue in either client repository, so the client handoff still needs issue creation there.
+The `rc` client currently has status, key list/status/create/delete/cancel-deletion, configure/reconfigure/start/restart/stop, and diagnostic/roundtrip entry points. It has no lifecycle enable/disable/rotate, key metadata, backup/restore, or reload commands; `POST /kms/reload` is the recovery path when a restarted server reports not-configured while a persisted configuration exists, so it is a client delivery item alongside the lifecycle gaps. The console currently calls service-status, configure/reconfigure/start/stop/config, clear-cache, status, and the modern key CRUD routes. It has no lifecycle, metadata, or backup/restore UI. These pending cells are delivery items for `rustfs/cli` and `rustfs/console`; they are not implemented in this repository. A read-only issue search on 2026-08-02 found no matching KMS issue in either client repository, so the client handoff still needs issue creation there.
 
 `POST /kms/generate-data-key` is deliberately marked “do not expose” for both clients: its response contains a base64 plaintext data key. `GET /kms/config` and backup status/restore responses contain operational paths and identifiers, not key material, but still require UI/CLI redaction and confirmation handling.
 

docs/operations/kms-observability-runbook.md

@@ -212,6 +212,17 @@ Investigation:
 
 Related signals: `rotation_due` / `rotation_due_reason` on the key listing; `rustfs_kms_deletion_sweep_keys_total{outcome=~"unreadable|failed"}` (a frozen gauge is stale, not healthy); the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) and pre-rotation checklist in the backend security properties document.
 
+## Startup persisted-configuration load
+
+KMS configured through the admin API is persisted to cluster storage and restored on every startup. The load result is visible in two places; check both before concluding that KMS "was never configured":
+
+- **Startup log**, `event="kms_persisted_config_lookup"` (`target: rustfs::init`): `state="found"` means the persisted configuration was loaded and applied; `state="not_found"` means no persisted configuration exists on disk; `state="load_failed"` means one exists but reading, unsealing, or decoding it failed.
+- **`GET /rustfs/admin/v3/kms/service-status`**: `"NotConfigured"` matches `not_found` (nothing persisted — configuring from scratch is the correct response), while a status of `Error("Failed to load persisted KMS configuration: ...")` or `Error("Failed to apply persisted KMS configuration: ...")` matches `load_failed`. The two states call for different operator actions; do not resubmit a full configuration to recover from `load_failed`.
+
+To recover from `load_failed` — or from any state where the server runs but its in-memory KMS lags the persisted configuration — call `POST /rustfs/admin/v3/kms/reload` (requires `kms:ServiceControl`). It re-reads the persisted configuration from cluster storage and reconfigures the service without resubmitting secrets, then broadcasts the reload to peer nodes. If reload keeps failing, check cluster storage health first (the read needs quorum), then `RUSTFS_KMS_CONFIG_SECRET`: an unseal error means the secret is missing or differs from the one that sealed the persisted copy — it must be identical on every node.
+
+A separate event, `kms_config_load_skipped` with `reason="storage_uninitialized"`, comes from the ambient loader used by the peer-reload RPC path; during normal startup the loader receives the store explicitly, so seeing this event outside a peer reload indicates a request arrived before storage initialization finished.
+
 ## Threshold calibration
 
 Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. `KmsKeyRotationOverdue` is different in the other direction: its 400-day threshold is a policy default (sitting above a common one-year rotation period), not a traffic default — calibrate it against the rotation period your compliance policy requires and against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`, not against a staging baseline. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).

rustfs/src/admin/handlers/kms_audit.rs

@@ -101,6 +101,8 @@ pub(super) enum KmsAdminOperation {
     Configure,
     /// Replacement of the running configuration.
     Reconfigure,
+    /// Reload of the persisted configuration.
+    Reload,
     /// Start or restart of the KMS service.
     Start,
     /// Stop of the KMS service.
@@ -127,6 +129,7 @@ impl KmsAdminOperation {
             Self::UntagResource => "UntagResource",
             Self::Configure => "Configure",
             Self::Reconfigure => "Reconfigure",
+            Self::Reload => "Reload",
             Self::Start => "Start",
             Self::Stop => "Stop",
             Self::Backup => "Backup",
@@ -147,7 +150,7 @@ impl KmsAdminOperation {
             // operations touch neither key material nor key state. Consumers
             // separate them from a plain access by the recorded operation name.
             Self::UpdateKeyDescription | Self::TagResource | Self::UntagResource => EventName::KmsKeyAccessed,
-            Self::Configure | Self::Reconfigure => EventName::KmsServiceConfigured,
+            Self::Configure | Self::Reconfigure | Self::Reload => EventName::KmsServiceConfigured,
             Self::Start => EventName::KmsServiceStarted,
             Self::Stop => EventName::KmsServiceStopped,
             // A backup reads the material of every key, and a restore

rustfs/src/admin/handlers/kms_dynamic.rs

@@ -18,11 +18,14 @@ use super::kms_audit::{KmsAdminAudit, KmsAdminOperation};
 use crate::admin::auth::validate_admin_request;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::{
-    current_app_context, current_kms_runtime_service_manager, current_notification_system_for_context,
-    current_object_store_handle_for_context, current_or_init_kms_runtime_service_manager,
+    AppContext, app_context_from_req, current_app_context, current_kms_runtime_service_manager,
+    current_notification_system_for_context, current_object_store_handle_for_context,
+    current_or_init_kms_runtime_service_manager,
 };
 use crate::admin::storage_api::config::{read_admin_config, save_admin_config};
 use crate::admin::storage_api::error::StorageError;
+use crate::admin::storage_api::runtime::ECStore;
+use crate::admin::storage_api::s3::{S3ErrorCode, error as admin_s3_error};
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
 use hyper::{Method, StatusCode};
@@ -35,6 +38,8 @@ use rustfs_kms::{
 use rustfs_policy::policy::action::{Action, KmsAction};
 use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use sha2::{Digest, Sha256};
+use std::future::Future;
+use std::sync::Arc;
 use tracing::{error, info, instrument, warn};
 
 /// Path to store KMS configuration in the cluster metadata
@@ -266,47 +271,48 @@ fn decode_persisted_kms_config(data: &[u8]) -> serde_json::Result<(KmsConfig, bo
     Ok((config, uses_legacy_local_defaults))
 }
 
-/// Load KMS configuration from cluster storage
-#[instrument]
-pub async fn load_kms_config() -> Option<KmsConfig> {
-    let context = current_app_context();
-    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
-        warn!(
-            component = LOG_COMPONENT_ADMIN,
-            subsystem = LOG_SUBSYSTEM_KMS,
-            event = "kms_config_load_skipped",
-            reason = "storage_uninitialized",
-            result = "config_load_skipped",
-            "admin kms dynamic state"
-        );
-        return None;
-    };
+#[derive(Debug, thiserror::Error)]
+pub enum KmsConfigLoadError {
+    #[error("storage layer is not initialized")]
+    StorageUnavailable,
+    #[error("failed to read persisted KMS configuration: {0}")]
+    StorageRead(#[source] StorageError),
+    #[error("failed to unseal persisted KMS configuration: {0}")]
+    Unseal(String),
+    #[error("failed to decode persisted KMS configuration: {0}")]
+    Decode(#[source] serde_json::Error),
+}
 
-    match read_admin_config(store, KMS_CONFIG_PATH).await {
+async fn load_kms_config_with<Read, ReadFuture>(read: Read) -> Result<Option<KmsConfig>, KmsConfigLoadError>
+where
+    Read: FnOnce() -> ReadFuture,
+    ReadFuture: Future<Output = Result<Vec<u8>, StorageError>>,
+{
+    match read().await {
         Ok(data) => {
             let (data, unseal_outcome) =
                 match open_persisted_kms_config(&data, rustfs_kms::config_secret::config_secret_from_env().as_deref()) {
                     Ok(opened) => opened,
                     Err(e) => {
                         error!(
+                            event = "kms_config_unseal_failed",
                             component = LOG_COMPONENT_ADMIN,
                             subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_config_unseal_failed",
-                            storage_path = KMS_CONFIG_PATH,
                             result = "config_unseal_failed",
+                            storage_path = KMS_CONFIG_PATH,
                             error = %e,
                             "admin kms dynamic state"
                         );
-                        return None;
+                        return Err(KmsConfigLoadError::Unseal(e));
                     }
                 };
             if !unseal_outcome.plaintext.is_empty() {
                 warn!(
+                    event = "kms_config_secret_unset",
                     component = LOG_COMPONENT_ADMIN,
                     subsystem = LOG_SUBSYSTEM_KMS,
-                    event = "kms_config_secret_unset",
-                    exposed_fields = ?unseal_outcome.plaintext,
                     state = "loaded_plaintext_secrets",
+                    exposed_fields = ?unseal_outcome.plaintext,
                     "persisted KMS configuration carries cleartext secrets; set RUSTFS_KMS_CONFIG_SECRET on every node and re-save to seal them"
                 );
             }
@@ -314,35 +320,35 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
                 Ok((config, is_legacy_local)) => {
                     if is_legacy_local {
                         warn!(
+                            event = "kms_legacy_local_config_loaded",
                             component = LOG_COMPONENT_ADMIN,
                             subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_legacy_local_config_loaded",
-                            storage_path = KMS_CONFIG_PATH,
                             state = "legacy_config_accepted",
+                            storage_path = KMS_CONFIG_PATH,
                             "admin kms dynamic state"
                         );
                     }
                     info!(
+                        event = "kms_config_loaded",
                         component = LOG_COMPONENT_ADMIN,
                         subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_config_loaded",
-                        storage_path = KMS_CONFIG_PATH,
                         state = "config_loaded",
+                        storage_path = KMS_CONFIG_PATH,
                         "admin kms dynamic state"
                     );
-                    Some(config)
+                    Ok(Some(config))
                 }
                 Err(e) => {
                     error!(
+                        event = "kms_config_deserialize_failed",
                         component = LOG_COMPONENT_ADMIN,
                         subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_config_deserialize_failed",
-                        storage_path = KMS_CONFIG_PATH,
                         result = "config_deserialize_failed",
+                        storage_path = KMS_CONFIG_PATH,
                         error = %e,
                         "admin kms dynamic state"
                     );
-                    None
+                    Err(KmsConfigLoadError::Decode(e))
                 }
             }
         }
@@ -353,29 +359,55 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
             // volume, bucket) means degraded storage and must stay a warning.
             if matches!(e, StorageError::ConfigNotFound) {
                 info!(
+                    event = "kms_config_loaded",
                     component = LOG_COMPONENT_ADMIN,
                     subsystem = LOG_SUBSYSTEM_KMS,
-                    event = "kms_config_loaded",
                     state = "not_found",
                     storage_path = KMS_CONFIG_PATH,
                     "admin kms dynamic state"
                 );
+                Ok(None)
             } else {
                 warn!(
+                    event = "kms_config_load_failed",
                     component = LOG_COMPONENT_ADMIN,
                     subsystem = LOG_SUBSYSTEM_KMS,
-                    event = "kms_config_load_failed",
-                    storage_path = KMS_CONFIG_PATH,
                     result = "config_load_failed",
+                    storage_path = KMS_CONFIG_PATH,
                     error = %e,
                     "admin kms dynamic state"
                 );
+                Err(KmsConfigLoadError::StorageRead(e))
             }
-            None
         }
     }
 }
 
+/// Load KMS configuration through an explicitly initialized cluster store.
+#[instrument(skip(store))]
+pub async fn load_kms_config_from_store(store: Arc<ECStore>) -> Result<Option<KmsConfig>, KmsConfigLoadError> {
+    load_kms_config_with(|| read_admin_config(store, KMS_CONFIG_PATH)).await
+}
+
+/// Load KMS configuration through the running server context.
+#[instrument]
+pub async fn load_kms_config() -> Option<KmsConfig> {
+    let context = current_app_context();
+    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
+        warn!(
+            event = "kms_config_load_skipped",
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_KMS,
+            result = "config_load_skipped",
+            reason = "storage_uninitialized",
+            "admin kms dynamic state"
+        );
+        return None;
+    };
+
+    load_kms_config_from_store(store).await.ok().flatten()
+}
+
 fn redact_config_secrets(value: &mut serde_json::Value) {
     match value {
         serde_json::Value::Object(object) => {
@@ -468,22 +500,42 @@ fn kms_config_is_unchanged(current: &KmsConfig, candidate: &KmsConfig) -> bool {
 /// request broadcasts, so that a runtime change reaches every node instead of
 /// only the one that served the admin request.
 pub async fn reload_persisted_kms_config() -> Result<(), String> {
-    let Some(config) = load_kms_config().await else {
+    let context = current_app_context();
+    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
+        return Err(KmsConfigLoadError::StorageUnavailable.to_string());
+    };
+    reload_persisted_kms_config_from_store(store, kms_service_manager_from_context(), "peer_reload").await
+}
+
+async fn reload_persisted_kms_config_from_store(
+    store: Arc<ECStore>,
+    service_manager: Arc<rustfs_kms::KmsServiceManager>,
+    operation: &'static str,
+) -> Result<(), String> {
+    let config = match load_kms_config_from_store(store).await {
+        Ok(config) => config,
+        Err(err) => {
+            service_manager
+                .record_initialization_error(format!("Failed to load persisted KMS configuration: {err}"))
+                .await;
+            return Err(err.to_string());
+        }
+    };
+    let Some(config) = config else {
         return Err("no persisted KMS configuration is available".to_string());
     };
 
-    let service_manager = kms_service_manager_from_context();
     if service_manager
         .get_config()
         .await
         .is_some_and(|current| kms_config_is_unchanged(&current, &config))
     {
         info!(
+            event = "kms_service_state",
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_KMS,
-            event = "kms_service_state",
-            operation = "peer_reload",
             state = "already_current",
+            operation,
             "admin kms dynamic state"
         );
         return Ok(());
@@ -491,11 +543,11 @@ pub async fn reload_persisted_kms_config() -> Result<(), String> {
 
     service_manager.reconfigure(config).await.map_err(|err| {
         error!(
+            event = "kms_service_state",
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_KMS,
-            event = "kms_service_state",
-            operation = "peer_reload",
             state = "reload_failed",
+            operation,
             error = %err,
             "admin kms dynamic state"
         );
@@ -503,11 +555,11 @@ pub async fn reload_persisted_kms_config() -> Result<(), String> {
     })?;
 
     info!(
+        event = "kms_service_state",
         component = LOG_COMPONENT_ADMIN,
         subsystem = LOG_SUBSYSTEM_KMS,
-        event = "kms_service_state",
-        operation = "peer_reload",
         state = "reconfigured",
+        operation,
         "admin kms dynamic state"
     );
     Ok(())
@@ -521,7 +573,11 @@ pub async fn reload_persisted_kms_config() -> Result<(), String> {
 /// trade a bounded divergence window for an outage.
 async fn broadcast_kms_config_reload() -> Vec<String> {
     let context = current_app_context();
-    let Some(notification_sys) = current_notification_system_for_context(context.as_deref()) else {
+    broadcast_kms_config_reload_for_context(context.as_deref()).await
+}
+
+async fn broadcast_kms_config_reload_for_context(context: Option<&AppContext>) -> Vec<String> {
+    let Some(notification_sys) = current_notification_system_for_context(context) else {
         return Vec::new();
     };
 
@@ -599,6 +655,12 @@ pub fn register_kms_dynamic_route(r: &mut S3Router<AdminOperation>) -> std::io::
         AdminOperation(&ReconfigureKmsHandler {}),
     )?;
 
+    r.insert(
+        Method::POST,
+        format!("{}{}", ADMIN_PREFIX, "/v3/kms/reload").as_str(),
+        AdminOperation(&ReloadKmsHandler {}),
+    )?;
+
     Ok(())
 }
 
@@ -1012,6 +1074,104 @@ impl Operation for StopKmsHandler {
     }
 }
 
+/// Reload the cluster-persisted KMS configuration.
+pub struct ReloadKmsHandler;
+
+#[async_trait::async_trait]
+impl Operation for ReloadKmsHandler {
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let context = app_context_from_req(&req)
+            .ok_or_else(|| admin_s3_error(S3ErrorCode::ServiceUnavailable, "server context is not ready"))?;
+        let Some(cred) = req.credentials else {
+            return Err(admin_s3_error(S3ErrorCode::InvalidRequest, "authentication required"));
+        };
+
+        let (cred, owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
+        let audit = KmsAdminAudit::from_request(&req.extensions, &req.headers, &cred);
+
+        audit.gate_admin(
+            validate_admin_request(
+                &req.headers,
+                &cred,
+                owner,
+                false,
+                kms_service_control_actions(),
+                req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
+            )
+            .await,
+            KmsAdminOperation::Reload,
+            None,
+        )?;
+
+        info!(
+            event = "kms_service_state",
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_KMS,
+            state = "requested",
+            operation = "reload",
+            "admin kms dynamic state"
+        );
+
+        let service_manager = context.kms().handle();
+        let (success, message, status) =
+            match reload_persisted_kms_config_from_store(context.object_store(), service_manager.clone(), "admin_reload").await {
+                Ok(()) => {
+                    let unconverged = broadcast_kms_config_reload_for_context(Some(context.as_ref())).await;
+                    let status = service_manager.get_status().await;
+                    let (success, message) =
+                        local_success_with_peer_report("Persisted KMS configuration reloaded successfully", &unconverged);
+                    info!(
+                        event = "kms_service_state",
+                        component = LOG_COMPONENT_ADMIN,
+                        subsystem = LOG_SUBSYSTEM_KMS,
+                        state = "reloaded",
+                        operation = "reload",
+                        status = ?status,
+                        "admin kms dynamic state"
+                    );
+                    audit.finish(KmsAdminOperation::Reload, None, None);
+                    (success, message, status)
+                }
+                Err(err) => {
+                    let kms_error = rustfs_kms::KmsError::backend_error(&err);
+                    error!(
+                        event = "kms_service_state",
+                        component = LOG_COMPONENT_ADMIN,
+                        subsystem = LOG_SUBSYSTEM_KMS,
+                        state = "reload_failed",
+                        operation = "reload",
+                        error = %err,
+                        "admin kms dynamic state"
+                    );
+                    audit.finish(KmsAdminOperation::Reload, None, Some(&kms_error));
+                    let status = service_manager.get_status().await;
+                    (false, format!("Failed to reload persisted KMS configuration: {err}"), status)
+                }
+            };
+
+        let response = ConfigureKmsResponse {
+            success,
+            message,
+            status,
+        };
+        let json_response = serde_json::to_string(&response).map_err(|err| {
+            error!(
+                event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
+                component = LOG_COMPONENT_ADMIN,
+                subsystem = LOG_SUBSYSTEM_KMS,
+                result = "response_serialize_failed",
+                operation = "reload",
+                error = %err,
+                "admin kms dynamic state"
+            );
+            admin_s3_error(S3ErrorCode::InternalError, "failed to serialize KMS reload response")
+        })?;
+
+        Ok(S3Response::new((StatusCode::OK, Body::from(json_response))))
+    }
+}
+
 /// Get KMS status handler
 pub struct GetKmsStatusHandler;
 
@@ -1259,10 +1419,15 @@ impl Operation for ReconfigureKmsHandler {
 #[cfg(test)]
 mod tests {
     use super::{
-        decode_persisted_kms_config, ensure_kms_config_persistable, ensure_kms_request_persistable, kms_config_fingerprint,
-        kms_config_is_unchanged, kms_configure_actions, kms_service_control_actions, local_success_with_peer_report,
-        normalize_configure_request_secrets, open_persisted_kms_config, redacted_canonical_config, seal_persisted_kms_config,
+        KmsConfigLoadError, decode_persisted_kms_config, ensure_kms_config_persistable, ensure_kms_request_persistable,
+        kms_config_fingerprint, kms_config_is_unchanged, kms_configure_actions, kms_service_control_actions,
+        load_kms_config_with, local_success_with_peer_report, normalize_configure_request_secrets, open_persisted_kms_config,
+        redacted_canonical_config, register_kms_dynamic_route, seal_persisted_kms_config,
     };
+    use crate::admin::router::{AdminOperation, S3Router};
+    use crate::admin::storage_api::error::StorageError;
+    use crate::server::ADMIN_PREFIX;
+    use hyper::Method;
     use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
     use std::path::PathBuf;
     use tempfile::TempDir;
@@ -1287,6 +1452,41 @@ mod tests {
         assert_lacks_action(&kms_service_control_actions(), Action::AdminAction(AdminAction::ServerInfoAdminAction));
     }
 
+    #[test]
+    fn kms_reload_route_is_registered() {
+        let mut router: S3Router<AdminOperation> = S3Router::new(false);
+        register_kms_dynamic_route(&mut router).expect("register KMS dynamic routes");
+
+        assert!(router.contains_route(Method::POST, &format!("{ADMIN_PREFIX}/v3/kms/reload")));
+    }
+
+    #[tokio::test]
+    async fn persisted_config_loader_distinguishes_absence_from_storage_failure() {
+        let absent = load_kms_config_with(|| async { Err(StorageError::ConfigNotFound) })
+            .await
+            .expect("missing configuration is not a load failure");
+        assert!(absent.is_none());
+
+        let error = load_kms_config_with(|| async { Err(StorageError::FaultyDisk) })
+            .await
+            .expect_err("storage failure must not look like missing configuration");
+        assert!(matches!(error, KmsConfigLoadError::StorageRead(StorageError::FaultyDisk)));
+    }
+
+    #[tokio::test]
+    async fn persisted_config_loader_returns_decoded_configuration() {
+        let expected = aws_configure_request("us-east-1").to_kms_config();
+        let data = serde_json::to_vec(&expected).expect("serialize persisted KMS config");
+
+        let loaded = load_kms_config_with(|| async { Ok(data) })
+            .await
+            .expect("load persisted KMS config")
+            .expect("persisted KMS config exists");
+
+        assert_eq!(loaded.backend, expected.backend);
+        assert_eq!(loaded.default_key_id, expected.default_key_id);
+    }
+
     #[test]
     fn persisted_beta5_local_config_retains_legacy_development_mode() {
         let temp_dir = TempDir::new().expect("create legacy local KMS directory");

rustfs/src/admin/route_policy.rs

@@ -782,6 +782,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
     admin(HttpMethod::Post, "/rustfs/admin/v3/kms/configure", KMS_CONFIGURE, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/kms/start", KMS_SERVICE_CONTROL, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/kms/stop", KMS_SERVICE_CONTROL, RouteRiskLevel::High),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/kms/reload", KMS_SERVICE_CONTROL, RouteRiskLevel::High),
     admin(
         HttpMethod::Get,
         "/rustfs/admin/v3/kms/service-status",
@@ -1987,6 +1988,7 @@ mod tests {
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/clear-cache", KMS_CLEAR_CACHE);
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/configure", KMS_CONFIGURE);
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/start", KMS_SERVICE_CONTROL);
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/reload", KMS_SERVICE_CONTROL);
         assert_action(HttpMethod::Delete, "/rustfs/admin/v3/kms/keys/delete", KMS_DELETE_KEY);
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/keys/cancel-deletion", KMS_DELETE_KEY);
         assert_action(HttpMethod::Get, "/rustfs/admin/v3/kms/keys/{key_id}", KMS_DESCRIBE_KEY);

rustfs/src/admin/route_registration_test.rs

@@ -347,6 +347,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::POST, "/v3/kms/configure"),
         admin_route(Method::POST, "/v3/kms/start"),
         admin_route(Method::POST, "/v3/kms/stop"),
+        admin_route(Method::POST, "/v3/kms/reload"),
         admin_route(Method::GET, "/v3/kms/service-status"),
         admin_route(Method::POST, "/v3/kms/reconfigure"),
         admin_route(Method::POST, "/v3/kms/keys"),
@@ -1313,6 +1314,7 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::POST, &admin_path("/v3/kms/create-key"));
     assert_route(&router, Method::POST, &admin_path("/v3/kms/key/create"));
     assert_route(&router, Method::POST, &admin_path("/v3/kms/configure"));
+    assert_route(&router, Method::POST, &admin_path("/v3/kms/reload"));
     assert_route(&router, Method::GET, &admin_path("/v3/kms/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/kms/status"));
     assert_route(&router, Method::GET, &admin_path("/v3/kms/key/status"));

rustfs/src/admin/snapshots/rustfs__admin__kms_contract__tests__kms_admin_route_contract.snap

@@ -171,6 +171,12 @@ expression: kms_route_contract()
     "action": "kms:Configure",
     "risk": "High"
   },
+  {
+    "method": "POST",
+    "path": "/rustfs/admin/v3/kms/reload",
+    "action": "kms:ServiceControl",
+    "risk": "High"
+  },
   {
     "method": "POST",
     "path": "/rustfs/admin/v3/kms/restore",

rustfs/src/init.rs

@@ -19,6 +19,7 @@ use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOpera
 use crate::storage_api::startup::init::{
     get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
 };
+use crate::storage_api::startup::services::ECStore;
 use crate::storage_api::startup::sse::log_sse_kms_key_policy_mode;
 use crate::{admin, config, startup_runtime_sources, version};
 use rustfs_config::{
@@ -490,10 +491,11 @@ async fn configure_and_start_kms(
 /// cluster storage and starts the service if found.
 /// # Arguments
 /// * `config` - The application configuration options
+/// * `store` - The initialized cluster store used for persisted configuration
 ///
 /// Returns `std::io::Result<()>` indicating success or failure
-#[instrument(skip(config))]
-pub async fn init_kms_system(config: &config::Config) -> std::io::Result<()> {
+#[instrument(skip(config, store))]
+pub async fn init_kms_system(config: &config::Config, store: Arc<ECStore>) -> std::io::Result<()> {
     // Initialize global KMS service manager (starts in NotConfigured state)
     let service_manager = startup_runtime_sources::init_kms_service_manager();
 
@@ -546,21 +548,21 @@ pub async fn init_kms_system(config: &config::Config) -> std::io::Result<()> {
             "Loading persisted KMS configuration"
         );
 
-        if let Some(persisted_config) = admin::handlers::kms_dynamic::load_kms_config().await {
-            info!(
-                target: "rustfs::init",
-                event = "kms_persisted_config_lookup",
-                component = LOG_COMPONENT_INIT,
-                subsystem = LOG_SUBSYSTEM_KMS,
-                state = "found",
-                "Loaded persisted KMS configuration"
-            );
+        match admin::handlers::kms_dynamic::load_kms_config_from_store(store).await {
+            Ok(Some(persisted_config)) => {
+                info!(
+                    target: "rustfs::init",
+                    event = "kms_persisted_config_lookup",
+                    component = LOG_COMPONENT_INIT,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    state = "found",
+                    "Loaded persisted KMS configuration"
+                );
 
-            // Configure the KMS service with persisted config
-            match configure_and_start_kms(&service_manager, persisted_config, "persisted configuration").await {
-                Ok(()) => {}
-                Err(e) => {
-                    warn!(
+                if let Err(e) = configure_and_start_kms(&service_manager, persisted_config, "persisted configuration").await {
+                    let message = format!("Failed to apply persisted KMS configuration: {e}");
+                    service_manager.record_initialization_error(message.clone()).await;
+                    error!(
                         target: "rustfs::init",
                         event = "kms_service_state",
                         component = LOG_COMPONENT_INIT,
@@ -571,15 +573,29 @@ pub async fn init_kms_system(config: &config::Config) -> std::io::Result<()> {
                     );
                 }
             }
-        } else {
-            info!(
-                target: "rustfs::init",
-                event = "kms_persisted_config_lookup",
-                component = LOG_COMPONENT_INIT,
-                subsystem = LOG_SUBSYSTEM_KMS,
-                state = "not_found",
-                "No persisted KMS configuration found"
-            );
+            Ok(None) => {
+                info!(
+                    target: "rustfs::init",
+                    event = "kms_persisted_config_lookup",
+                    component = LOG_COMPONENT_INIT,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    state = "not_found",
+                    "No persisted KMS configuration found"
+                );
+            }
+            Err(e) => {
+                let message = format!("Failed to load persisted KMS configuration: {e}");
+                service_manager.record_initialization_error(message).await;
+                error!(
+                    target: "rustfs::init",
+                    event = "kms_persisted_config_lookup",
+                    component = LOG_COMPONENT_INIT,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    state = "load_failed",
+                    error = %e,
+                    "Persisted KMS configuration load failed"
+                );
+            }
         }
     }
 

rustfs/src/startup_embedded_optional.rs

@@ -16,15 +16,17 @@ use crate::{
     config::Config,
     init::{init_buffer_profile_system, init_kms_system},
     startup_audit::init_event_notifier_and_audit,
+    storage_api::startup::services::ECStore,
 };
+use std::sync::Arc;
 use tracing::warn;
 
 const LOG_COMPONENT_EMBEDDED: &str = "embedded";
 const LOG_SUBSYSTEM_EMBEDDED: &str = "embedded";
 const EVENT_EMBEDDED_OPTIONAL_SERVICE_SKIPPED: &str = "embedded_optional_service_skipped";
 
-pub(crate) async fn init_embedded_optional_service_runtime(config: &Config) {
-    if let Err(err) = init_kms_system(config).await {
+pub(crate) async fn init_embedded_optional_service_runtime(config: &Config, store: Arc<ECStore>) {
+    if let Err(err) = init_kms_system(config, store).await {
         log_embedded_optional_service_skipped("kms", err);
     }
 

rustfs/src/startup_services.rs

@@ -58,7 +58,7 @@ pub(crate) async fn init_embedded_startup_runtime_services(
     readiness: Arc<GlobalReadiness>,
     server_ctx: Arc<ServerContextSlot>,
 ) -> Result<EmbeddedStartupServiceRuntime> {
-    init_embedded_optional_service_runtime(config).await;
+    init_embedded_optional_service_runtime(config, store.clone()).await;
     let buckets = init_embedded_bucket_metadata_runtime(store.clone(), &ctx).await?;
     let iam_bootstrap = init_embedded_iam_runtime(store, ctx, readiness, server_ctx)
         .await
@@ -77,7 +77,7 @@ pub(crate) async fn init_startup_runtime_services(
     state_manager: Arc<ServiceStateManager>,
     server_ctx: Arc<ServerContextSlot>,
 ) -> Result<StartupServiceRuntime> {
-    init_kms_system(config).await?;
+    init_kms_system(config, store.clone()).await?;
 
     let optional_runtimes = init_optional_runtime_services().await?;
     let heartbeat_config = HeartbeatConfig::from_env().map_err(std::io::Error::other)?;