From 1266d4c2701f79a719652834c5cc4cd596cd7177 Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 08:53:11 +0800 Subject: [PATCH] feat(tls): reload and publish external certificate generations (#8467) * feat(tls): reload external certificate snapshots coherently Pin projected certificate generations, validate complete TLS material, coordinate server consumers, and invalidate outbound RPC caches coherently. Add externally managed TLS Secret support and rotation guidance. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * feat(tls): publish server certificates through a fenced admin API Add opt-in server-leaf publication with dedicated IAM actions, actual TLS transport checks, immutable generations, durable captured-token CAS and idempotent receipts. Preserve independent trust and client identity sources and distinguish visible publication state from the full serving chain. Validate 64 TLS tests, 41 TLS-runtime tests, the focused IAM policy test, and eight live local API scenarios including least privilege and restart. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(admin): route certificate handler types through S3 facade Use the existing admin S3 facade to keep direct s3s imports within the footprint ratchet. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(admin): include TLS certificate routes in route matrix Account for the registered certificate-state GET and certificate publication POST without changing route behavior or weakening matrix assertions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(tls): reconcile startup trust before skipping reloads Compare the initialized listener bundle with coherently published outbound material, retain startup trust and client identity when sources disappear, and cover independent startup rotations through real reload passes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(tls): reuse pinned generations and prepared listener state Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(tls): preserve startup trust source requirements --------- Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: overtrue --- Cargo.lock | 2 + crates/common/Cargo.toml | 1 + crates/common/src/globals.rs | 136 +- crates/ecstore/src/cluster/rpc/client.rs | 68 +- crates/ecstore/src/runtime/sources.rs | 5 +- crates/policy/src/policy/action.rs | 34 + crates/protos/src/lib.rs | 428 +++++- crates/protos/src/runtime_sources.rs | 4 + crates/tls-runtime/Cargo.toml | 1 + crates/tls-runtime/src/certs.rs | 208 ++- crates/tls-runtime/src/error.rs | 32 + crates/tls-runtime/src/lib.rs | 7 +- crates/tls-runtime/src/material.rs | 656 ++++++++- crates/tls-runtime/src/outbound.rs | 26 +- crates/tls-runtime/src/source.rs | 122 ++ docs/README.md | 2 + docs/operations/tls-certificate-rotation.md | 204 +++ helm/rustfs/templates/_helpers.tpl | 23 +- helm/rustfs/templates/configmap.yaml | 11 +- helm/rustfs/templates/deployment.yaml | 80 +- helm/rustfs/templates/statefulset.yaml | 82 +- helm/rustfs/values.yaml | 8 + rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/handlers/tls_certificate.rs | 177 +++ rustfs/src/admin/mod.rs | 1 + rustfs/src/admin/route_policy.rs | 22 + rustfs/src/admin/route_registration_test.rs | 2 + rustfs/src/server/http.rs | 42 +- rustfs/src/server/mod.rs | 2 + rustfs/src/server/ssec_transport.rs | 7 +- rustfs/src/server/tls_material.rs | 1383 +++++++++++++----- rustfs/src/server/tls_publish.rs | 1341 +++++++++++++++++ rustfs/src/startup_runtime_sources.rs | 1 + rustfs/src/startup_tls_material.rs | 28 + scripts/test_helm_templates.sh | 7 +- scripts/test_helm_tls_reload.sh | 49 + 36 files changed, 4594 insertions(+), 609 deletions(-) create mode 100644 docs/operations/tls-certificate-rotation.md create mode 100644 rustfs/src/admin/handlers/tls_certificate.rs create mode 100644 rustfs/src/server/tls_publish.rs create mode 100644 scripts/test_helm_tls_reload.sh diff --git a/Cargo.lock b/Cargo.lock index 44433ab1e..91b1a5149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9686,6 +9686,7 @@ dependencies = [ name = "rustfs-common" version = "1.0.1" dependencies = [ + "arc-swap", "hotpath", "metrics", "smallvec", @@ -10957,6 +10958,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", + "x509-parser", ] [[package]] diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 3749c929d..b772c459d 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -34,6 +34,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] +arc-swap.workspace = true hotpath.workspace = true tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } tonic = { workspace = true, features = ["gzip", "deflate"] } diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index e4cd89c88..03814d454 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use arc_swap::ArcSwap; use std::collections::HashMap; -use std::sync::LazyLock; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock}; use tokio::sync::RwLock; use tonic::transport::Channel; @@ -27,6 +28,42 @@ pub static GLOBAL_ROOT_CERT: LazyLock>>> = LazyLock::new(| pub static GLOBAL_MTLS_IDENTITY: LazyLock>> = LazyLock::new(|| RwLock::new(None)); pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock = LazyLock::new(|| AtomicU64::new(0)); +#[derive(Clone, Default)] +pub struct OutboundTlsSnapshot { + pub generation: u64, + pub root_ca_pem: Option>, + pub mtls_identity: Option, +} + +static GLOBAL_OUTBOUND_TLS_STATE: LazyLock> = + LazyLock::new(|| ArcSwap::from_pointee(OutboundTlsSnapshot::default())); + +pub fn get_global_outbound_tls_snapshot() -> Arc { + GLOBAL_OUTBOUND_TLS_STATE.load_full() +} + +/// Publish identity, trust and generation together; the individual globals are legacy views. +pub async fn publish_global_outbound_tls_snapshot(snapshot: OutboundTlsSnapshot) -> u64 { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + if current.generation != 0 + && current.root_ca_pem == snapshot.root_ca_pem + && current.mtls_identity == snapshot.mtls_identity + { + return Arc::clone(current); + } + Arc::new(OutboundTlsSnapshot { + generation: snapshot.generation.max(current.generation.saturating_add(1)), + root_ca_pem: snapshot.root_ca_pem.clone(), + mtls_identity: snapshot.mtls_identity.clone(), + }) + }); + let accepted = get_global_outbound_tls_snapshot(); + *GLOBAL_ROOT_CERT.write().await = accepted.root_ca_pem.clone(); + *GLOBAL_MTLS_IDENTITY.write().await = accepted.mtls_identity.clone(); + GLOBAL_OUTBOUND_TLS_GENERATION.store(accepted.generation, Ordering::Relaxed); + accepted.generation +} + /// Log level to use when reporting cached gRPC connection eviction. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ConnectionEvictionLogLevel { @@ -85,17 +122,27 @@ pub async fn get_global_addr() -> String { /// # Arguments /// * `cert` - A vector of bytes representing the PEM-encoded root CA certificate. pub async fn set_global_root_cert(cert: Vec) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.root_ca_pem = Some(cert.clone()); + Arc::new(next) + }); *GLOBAL_ROOT_CERT.write().await = Some(cert); } /// Clear the global root CA certificate for outbound gRPC clients. pub async fn clear_global_root_cert() { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.root_ca_pem = None; + Arc::new(next) + }); *GLOBAL_ROOT_CERT.write().await = None; } /// Get the global root CA certificate for outbound gRPC clients. pub async fn get_global_root_cert() -> Option> { - GLOBAL_ROOT_CERT.read().await.clone() + get_global_outbound_tls_snapshot().root_ca_pem.clone() } /// Set the global mTLS identity (cert+key PEM) for outbound gRPC clients. @@ -105,22 +152,32 @@ pub async fn get_global_root_cert() -> Option> { /// # Arguments /// * `identity` - An optional MtlsIdentityPem struct containing the cert and key PEM. pub async fn set_global_mtls_identity(identity: Option) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.mtls_identity = identity.clone(); + Arc::new(next) + }); *GLOBAL_MTLS_IDENTITY.write().await = identity; } /// Get the global mTLS identity for outbound gRPC clients. pub async fn get_global_mtls_identity() -> Option { - GLOBAL_MTLS_IDENTITY.read().await.clone() + get_global_outbound_tls_snapshot().mtls_identity.clone() } /// Set the global outbound TLS generation. pub fn set_global_outbound_tls_generation(generation: u64) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.generation = generation; + Arc::new(next) + }); GLOBAL_OUTBOUND_TLS_GENERATION.store(generation, Ordering::Relaxed); } /// Get the global outbound TLS generation. pub fn get_global_outbound_tls_generation() -> u64 { - GLOBAL_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed) + get_global_outbound_tls_snapshot().generation } /// Evict a stale/dead connection from the global connection cache. @@ -194,8 +251,77 @@ pub async fn clear_all_connections() { /// /// When present, gRPC clients will present this identity to servers requesting/requiring mTLS. /// When absent, clients proceed with standard server-authenticated TLS. -#[derive(Clone, Debug)] +#[derive(Clone, PartialEq, Eq)] pub struct MtlsIdentityPem { pub cert_pem: Vec, pub key_pem: Vec, } + +impl std::fmt::Debug for MtlsIdentityPem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MtlsIdentityPem") + .field("cert_bytes", &self.cert_pem.len()) + .field("key_pem", &"[REDACTED]") + .finish() + } +} + +#[cfg(test)] +mod tls_snapshot_tests { + use super::*; + + #[tokio::test] + async fn outbound_snapshot_publication_is_coherent_and_skips_unchanged_material() { + let start = get_global_outbound_tls_generation().saturating_add(1); + let snapshot = OutboundTlsSnapshot { + generation: start, + root_ca_pem: Some(vec![1]), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: vec![1], + key_pem: vec![1], + }), + }; + let first = publish_global_outbound_tls_snapshot(snapshot.clone()).await; + assert_eq!( + publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: first.saturating_add(1), + ..snapshot + }) + .await, + first + ); + let writer = tokio::spawn(async move { + for tag in 2_u8..=255 { + publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: start.saturating_add(u64::from(tag)), + root_ca_pem: Some(vec![tag]), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: vec![tag], + key_pem: vec![tag], + }), + }) + .await; + tokio::task::yield_now().await; + } + }); + while !writer.is_finished() { + let state = get_global_outbound_tls_snapshot(); + let identity = state.mtls_identity.as_ref().expect("identity in complete snapshot"); + assert_eq!(state.root_ca_pem.as_ref().expect("roots in complete snapshot"), &identity.cert_pem); + assert_eq!(identity.cert_pem, identity.key_pem); + tokio::task::yield_now().await; + } + writer.await.expect("writer completes"); + } + + #[test] + fn mtls_identity_debug_redacts_private_key() { + let identity = MtlsIdentityPem { + cert_pem: b"public certificate".to_vec(), + key_pem: b"PRIVATE-KEY-MARKER".to_vec(), + }; + let debug = format!("{identity:?}"); + assert!(!debug.contains("PRIVATE-KEY-MARKER")); + assert!(debug.contains("[REDACTED]")); + } +} diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 76ae3bb36..27bd0542f 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -24,10 +24,11 @@ use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_he #[cfg(test)] use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers}; use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError}; +#[cfg(test)] use crate::runtime::sources as runtime_sources; use http::{Request as HttpRequest, Response as HttpResponse, Uri}; use rustfs_protos::{ - ChannelClass, create_new_channel, get_channel_for_class, + ChannelClass, get_channel_for_class, proto_gen::node_service::{ heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, scanner_control_service_client::ScannerControlServiceClient, @@ -66,12 +67,9 @@ pub(crate) async fn scanner_control_time_out_client( interceptor: TonicInterceptor, ) -> crate::error::Result>> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr) - .await - .map_err(|err| crate::error::Error::other(err.to_string()))?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control) + .await + .map_err(|err| crate::error::Error::other(err.to_string()))?; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor) @@ -84,10 +82,7 @@ pub async fn heal_control_time_out_client( interceptor: TonicInterceptor, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr).await?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control).await?; let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); Ok(HealControlServiceClient::with_interceptor(channel, interceptor) @@ -100,10 +95,7 @@ pub async fn tier_mutation_control_time_out_client( interceptor: TonicInterceptor, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr).await?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control).await?; let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor) @@ -123,17 +115,7 @@ pub async fn node_service_time_out_client_for_class( class: ChannelClass, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match class { - ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await { - Some(channel) => { - debug!("Using cached gRPC channel for: {}", addr); - channel - } - // No cached connection, create new one. - None => create_new_channel(addr).await?, - }, - ChannelClass::Bulk => get_channel_for_class(addr, ChannelClass::Bulk).await?, - }; + let channel = get_channel_for_class(addr, class).await?; let max_message_size = rustfs_protos::internode_rpc_max_message_size(); let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); @@ -546,6 +528,40 @@ mod tests { use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{Registry, layer::SubscriberExt}; + #[tokio::test] + async fn rpc_client_factories_do_not_reuse_https_channels_without_tls_generation() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve local address"); + let addr = format!("https://{}", listener.local_addr().expect("local address")); + drop(listener); + for factory in 0..4 { + let placeholder = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + runtime_sources::cache_test_node_channel(addr.clone(), placeholder).await; + let interceptor = TonicInterceptor::NoOp(NoOpInterceptor); + let _result = match factory { + 0 => scanner_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + 1 => heal_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + 2 => tier_mutation_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + _ => node_service_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + }; + assert!( + !runtime_sources::test_node_channel_is_cached(&addr).await, + "factory {factory} must evict an unbound HTTPS cache entry before attempting a fresh dial" + ); + } + } + #[derive(Clone)] struct EpochProofService { audience: String, diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index f121cffe5..b7db108bf 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -50,6 +50,7 @@ use s3s::dto::BucketLifecycleConfiguration; use s3s::region::Region; use tokio::sync::{OwnedRwLockReadGuard, RwLock}; use tokio_util::sync::CancellationToken; +#[cfg(test)] use tonic::transport::Channel; use uuid::Uuid; @@ -251,10 +252,6 @@ pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option { } } -pub(crate) async fn cached_node_channel(addr: &str) -> Option { - rustfs_common::cached_connection(addr).await -} - #[cfg(test)] pub(crate) async fn cache_test_node_channel(addr: String, channel: Channel) { rustfs_common::cache_connection(addr, channel).await; diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index e2c7742d5..d86952566 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -447,6 +447,10 @@ pub enum AdminAction { ServiceFreezeAdminAction, #[strum(serialize = "admin:ConfigUpdate")] ConfigUpdateAdminAction, + #[strum(serialize = "admin:TLSCertificatePublish")] + TLSCertificatePublishAdminAction, + #[strum(serialize = "admin:TLSCertificateStatus")] + TLSCertificateStatusAdminAction, #[strum(serialize = "admin:CreateUser")] CreateUserAdminAction, #[strum(serialize = "admin:DeleteUser")] @@ -646,6 +650,8 @@ impl AdminAction { | AdminAction::ServiceStopAdminAction | AdminAction::ServiceFreezeAdminAction | AdminAction::ConfigUpdateAdminAction + | AdminAction::TLSCertificatePublishAdminAction + | AdminAction::TLSCertificateStatusAdminAction | AdminAction::CreateUserAdminAction | AdminAction::DeleteUserAdminAction | AdminAction::ListUsersAdminAction @@ -941,6 +947,34 @@ mod tests { assert!(!Action::AdminAction(get_action).is_match(&Action::AdminAction(set_action))); } + #[test] + fn tls_certificate_actions_preserve_least_privilege() { + let publish = Action::try_from("admin:TLSCertificatePublish").expect("parse certificate publish action"); + let status = Action::try_from("admin:TLSCertificateStatus").expect("parse certificate status action"); + for (action, name) in [ + (&publish, "admin:TLSCertificatePublish"), + (&status, "admin:TLSCertificateStatus"), + ] { + let Action::AdminAction(admin_action) = action else { + panic!("certificate policy must use an admin action"); + }; + assert!(admin_action.is_valid()); + assert_eq!(serde_json::to_string(action).expect("serialize action"), format!("\"{name}\"")); + } + assert!(!status.is_match(&publish)); + assert!(!publish.is_match(&status)); + let config = Action::AdminAction(AdminAction::ConfigUpdateAdminAction); + let server_info = Action::AdminAction(AdminAction::ServerInfoAdminAction); + assert!(!config.is_match(&publish)); + assert!(!server_info.is_match(&publish)); + let admin = Action::AdminAction(AdminAction::AllAdminActions); + assert!(admin.is_match(&publish)); + assert!(admin.is_match(&status)); + let s3 = Action::try_from("s3:*").expect("parse S3 wildcard"); + assert!(!s3.is_match(&publish)); + assert!(!s3.is_match(&status)); + } + #[test] fn test_table_catalog_admin_action_is_valid() { let get_action = AdminAction::try_from("admin:GetTableCatalog").expect("Should parse GetTableCatalog action"); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index a521c3a9f..451afa21a 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -25,8 +25,8 @@ use rustfs_common::{cache_connection, cached_connection, evict_connection_with_l use std::{ collections::HashMap, error::Error, - sync::LazyLock, sync::atomic::{AtomicU8, AtomicUsize, Ordering}, + sync::{Arc, LazyLock, Mutex as StdMutex, Weak}, time::{Duration, Instant}, }; use tokio::sync::Mutex; @@ -59,6 +59,44 @@ const INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET: u8 = 0; const INTERNODE_RPC_MSGPACK_ONLY_CACHE_FALSE: u8 = 1; const INTERNODE_RPC_MSGPACK_ONLY_CACHE_TRUE: u8 = 2; static TLS_GENERATION_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +// Lock order: generation metadata, then the common connection cache. The dial +// registry's standard mutex is released before waiting on any async lock. +static DIAL_LOCK_REGISTRY: LazyLock>>>> = LazyLock::new(|| StdMutex::new(HashMap::new())); + +struct DialLock { + key: String, + mutex: Arc>, +} + +impl DialLock { + fn for_key(key: &str) -> Self { + let mut registry = DIAL_LOCK_REGISTRY.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + registry.retain(|_, lock| lock.strong_count() != 0); + let mutex = registry.get(key).and_then(Weak::upgrade).unwrap_or_else(|| { + let mutex = Arc::new(Mutex::new(())); + registry.insert(key.to_string(), Arc::downgrade(&mutex)); + mutex + }); + Self { + key: key.to_string(), + mutex, + } + } +} + +impl Drop for DialLock { + fn drop(&mut self) { + let mut registry = DIAL_LOCK_REGISTRY.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if Arc::strong_count(&self.mutex) == 1 + && registry + .get(&self.key) + .is_some_and(|slot| Weak::ptr_eq(slot, &Arc::downgrade(&self.mutex))) + { + registry.remove(&self.key); + } + } +} + static INTERNODE_RPC_MSGPACK_ONLY_CACHE: AtomicU8 = AtomicU8::new(INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET); fn enforce_tls_generation_cache_bound(generation_cache: &mut HashMap, generation: u64, addr: &str) { @@ -2805,6 +2843,53 @@ fn bulk_cache_key(addr: &str, idx: usize) -> String { format!("{addr}\u{0}bulk\u{0}{idx}") } +async fn cached_channel_for_tls_generation(dial_addr: &str, cache_key: &str) -> Option { + if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) { + return cached_connection(cache_key).await; + } + let mut generations = TLS_GENERATION_CACHE.lock().await; + let Some(channel) = cached_connection(cache_key).await else { + generations.remove(cache_key); + return None; + }; + if generations.get(cache_key).copied() == Some(runtime_sources::outbound_tls_generation()) { + return Some(channel); + } + evict_connection_with_log_level(cache_key, ConnectionEvictionLogLevel::Debug).await; + generations.remove(cache_key); + runtime_sources::record_stale_grpc_channel_tls_generation(); + None +} + +async fn cache_dialed_channel(dial_addr: &str, cache_key: &str, channel: &Channel, generation: Option) -> bool { + if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) { + cache_connection(cache_key.to_string(), channel.clone()).await; + return true; + } + let mut generations = TLS_GENERATION_CACHE.lock().await; + if generation != Some(runtime_sources::outbound_tls_generation()) { + runtime_sources::record_stale_grpc_channel_tls_generation(); + return false; + } + cache_connection(cache_key.to_string(), channel.clone()).await; + // Publication can advance while the common cache's async lock is awaited. + if generation != Some(runtime_sources::outbound_tls_generation()) { + evict_connection_with_log_level(cache_key, ConnectionEvictionLogLevel::Debug).await; + generations.remove(cache_key); + runtime_sources::record_stale_grpc_channel_tls_generation(); + return false; + } + if let Some(generation) = generation { + enforce_tls_generation_cache_bound(&mut generations, generation, cache_key); + generations.insert(cache_key.to_string(), generation); + } + true +} + +fn channel_acquisition_timeout(_: tokio::time::error::Elapsed) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::TimedOut, "internode channel acquisition deadline elapsed") +} + /// Acquire a cached-or-newly-dialed channel for the given peer and channel class. /// /// For [`ChannelClass::Control`] (or whenever isolation is disabled) this is exactly the legacy @@ -2813,22 +2898,27 @@ fn bulk_cache_key(addr: &str, idx: usize) -> String { /// per-peer bulk pool and dialed on demand, physically isolating large transfers from /// control-plane RPCs. pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result> { - if class == ChannelClass::Control || !channel_isolation_enabled() { - if let Some(channel) = cached_connection(addr).await { - debug!("Using cached control gRPC channel for: {}", addr); + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + tokio::time::timeout_at(deadline, async { + if class == ChannelClass::Control || !channel_isolation_enabled() { + if let Some(channel) = cached_channel_for_tls_generation(addr, addr).await { + debug!("Using cached control gRPC channel for: {}", addr); + return Ok(channel); + } + return build_channel_until(addr, addr, deadline).await; + } + + let pool_size = bulk_channel_pool_size(); + let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size; + let cache_key = bulk_cache_key(addr, idx); + if let Some(channel) = cached_channel_for_tls_generation(addr, &cache_key).await { + debug!("Using cached bulk gRPC channel {} for: {}", idx, addr); return Ok(channel); } - return create_new_channel(addr).await; - } - - let pool_size = bulk_channel_pool_size(); - let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size; - let cache_key = bulk_cache_key(addr, idx); - if let Some(channel) = cached_connection(&cache_key).await { - debug!("Using cached bulk gRPC channel {} for: {}", idx, addr); - return Ok(channel); - } - build_channel(addr, &cache_key).await + build_channel_until(addr, &cache_key, deadline).await + }) + .await + .map_err(channel_acquisition_timeout)? } /// Creates a new gRPC channel with optimized keepalive settings for cluster resilience. @@ -2852,6 +2942,45 @@ pub async fn create_new_channel(addr: &str) -> Result> { /// for isolated bulk channels (see [`bulk_cache_key`]), letting several physically distinct /// channels to the same peer be cached independently. async fn build_channel(dial_addr: &str, cache_key: &str) -> Result> { + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + build_channel_until(dial_addr, cache_key, deadline).await +} + +async fn build_channel_until( + dial_addr: &str, + cache_key: &str, + deadline: tokio::time::Instant, +) -> Result> { + tokio::time::timeout_at(deadline, async { + if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { + return Ok(channel); + } + let slot = DialLock::for_key(cache_key); + let _guard = slot.mutex.lock().await; + loop { + // A successful concurrent miss has already populated this exact key. + if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { + return Ok(channel); + } + let (channel, generation) = dial_channel(dial_addr, cache_key).await?; + if cache_dialed_channel(dial_addr, cache_key, &channel, generation).await + && let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await + { + debug!( + "Successfully created and cached gRPC channel to: {} (cache key: {})", + dial_addr, cache_key + ); + return Ok(channel); + } + // A generation published during dial cannot be returned; rebuilding + // shares the original deadline with lock wait and all earlier dials. + } + }) + .await + .map_err(channel_acquisition_timeout)? +} + +async fn dial_channel(dial_addr: &str, cache_key: &str) -> Result<(Channel, Option), Box> { debug!("Creating new gRPC channel to: {} (cache key: {})", dial_addr, cache_key); let dial_started_at = Instant::now(); let connect_timeout = internode_connect_timeout(); @@ -2887,18 +3016,10 @@ async fn build_channel(dial_addr: &str, cache_key: &str) -> Result Result = tokio::sync::Mutex::const_new(()); + static INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK: Mutex<()> = Mutex::new(()); #[derive(Clone, PartialEq, prost::Message)] @@ -3144,6 +3253,243 @@ mod tests { assert_eq!(manifest_fields, proto_bin_json_fields("Response")); } + #[tokio::test] + async fn tls02_stale_https_control_and_bulk_cache_entries_are_evicted() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-stale-cache.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + let stale_generation = generation.checked_add(1).unwrap_or(0); + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.insert(key.clone(), stale_generation); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_none(), + "TLS02_INVARIANT stale HTTPS cache entry cannot be returned" + ); + assert!( + !rustfs_common::has_cached_connection(&key).await, + "TLS02_INVARIANT stale entry is removed from the real connection cache" + ); + } + } + + #[tokio::test] + async fn tls02_missing_https_generation_control_and_bulk_cache_entries_are_evicted() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-missing-generation.invalid:9000"; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.remove(&key); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_none(), + "TLS02_INVARIANT HTTPS channel without a generation cannot be returned" + ); + assert!( + !rustfs_common::has_cached_connection(&key).await, + "TLS02_INVARIANT unbound entry is removed from the real connection cache" + ); + } + } + + #[tokio::test] + async fn tls02_current_https_control_and_bulk_cache_entries_are_reused() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-current-generation.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.insert(key.clone(), generation); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTPS channel at the canonical generation is reused" + ); + assert!(rustfs_common::has_cached_connection(&key).await); + evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await; + TLS_GENERATION_CACHE.lock().await.remove(&key); + } + } + + #[tokio::test] + async fn tls02_http_control_and_bulk_cache_ignore_tls_generation() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "http://tls02-http-cache.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE + .lock() + .await + .insert(key.clone(), generation.checked_add(1).unwrap_or(0)); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTP cache is independent of TLS generations" + ); + TLS_GENERATION_CACHE.lock().await.remove(&key); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTP cache needs no TLS generation metadata" + ); + evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await; + } + } + + #[tokio::test] + async fn tls02_captured_identity_is_immutable_and_late_dial_keeps_its_actual_generation() { + use rustfs_common::MtlsIdentityPem; + use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration, publish_global_outbound_tls_state}; + let _test_guard = TLS02_TEST_LOCK.lock().await; + let previous = runtime_sources::outbound_tls_state().await; + let first_generation = previous.generation.0.checked_add(1).expect("test generation capacity"); + let first = OutboundTlsMaterial { + root_ca_pem: b"root-a".to_vec(), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: b"certificate-a".to_vec(), + key_pem: b"test-key-a".to_vec(), + }), + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation), &first).await; + let captured = runtime_sources::outbound_tls_state().await; + let second = OutboundTlsMaterial { + root_ca_pem: b"root-b".to_vec(), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: b"certificate-b".to_vec(), + key_pem: b"test-key-b".to_vec(), + }), + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation + 1), &second).await; + let current = runtime_sources::outbound_tls_state().await; + let addr = "https://tls02-late-dial.invalid:9000"; + let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + let cached = cache_dialed_channel(addr, addr, &channel, Some(captured.generation.0)).await; + let original_key = captured.mtls_identity.as_ref().expect("captured identity").key_pem.clone(); + let current_key = current.mtls_identity.as_ref().expect("current identity").key_pem.clone(); + let restore = OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation + 2), &restore).await; + assert_eq!( + original_key, b"test-key-a", + "TLS02_INVARIANT captured identity cannot change with publication" + ); + assert_eq!(captured.root_ca_pem.as_deref(), Some(b"root-a".as_slice())); + assert_eq!(current_key, b"test-key-b"); + assert!(!cached, "TLS02_INVARIANT late dial cannot be relabeled with the current generation"); + assert!( + !rustfs_common::has_cached_connection(addr).await, + "TLS02_INVARIANT late stale dial is never cached" + ); + assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(addr)); + } + + #[tokio::test] + async fn tls02_concurrent_control_and_bulk_misses_merge_successful_real_dials() { + use tonic::codegen::tokio_stream::{StreamExt, wrappers::TcpListenerStream}; + let _test_guard = TLS02_TEST_LOCK.lock().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind HTTP/2 test server"); + let addr = format!("http://{}", listener.local_addr().expect("test server address")); + let accepted = Arc::new(AtomicUsize::new(0)); + let counter = accepted.clone(); + let incoming = TcpListenerStream::new(listener).map(move |stream| { + if stream.is_ok() { + counter.fetch_add(1, Ordering::SeqCst); + } + stream + }); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_routes(tonic::service::Routes::default()) + .serve_with_incoming(incoming) + .await + }); + for key in [addr.clone(), bulk_cache_key(&addr, 0)] { + let mut tasks = Vec::new(); + for _ in 0..32 { + let dial_addr = addr.clone(); + let cache_key = key.clone(); + tasks.push(tokio::spawn(async move { + build_channel(&dial_addr, &cache_key).await.map_err(|error| error.to_string()) + })); + } + for task in tasks { + let channel = task.await.expect("channel acquisition task").expect("real HTTP/2 dial"); + drop(channel); + } + assert!(rustfs_common::has_cached_connection(&key).await); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key), + "TLS02_INVARIANT completed miss groups leave no lock entries" + ); + } + // Tonic returns its channel before the server task necessarily polls the + // already-established sockets out of the listener's accept queue. + tokio::time::sleep(Duration::from_millis(20)).await; + let connections = accepted.load(Ordering::SeqCst); + evict_connection_with_log_level(&addr, ConnectionEvictionLogLevel::Debug).await; + evict_connection_with_log_level(&bulk_cache_key(&addr, 0), ConnectionEvictionLogLevel::Debug).await; + server.abort(); + let _ = server.await; + assert_eq!(connections, 2, "TLS02_INVARIANT 32 misses per control/bulk key dial once each"); + } + + #[tokio::test] + async fn tls02_concurrent_waiters_share_bounded_total_deadlines_and_release_registry() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "http://tls02-contended-dial.invalid:9000"; + let owner = DialLock::for_key(addr); + let guard = owner.mutex.lock().await; + let mut tasks = Vec::new(); + for _ in 0..8 { + tasks.push(tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_millis(40); + let result = build_channel_until(addr, addr, deadline).await; + result.err().is_some_and(|error| { + error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::TimedOut) + }) + })); + } + let completed = tokio::time::timeout(Duration::from_millis(250), async { + for task in tasks { + assert!( + task.await.expect("deadline waiter"), + "TLS02_INVARIANT wait and rebuild share a typed timeout" + ); + } + }) + .await; + drop(guard); + drop(owner); + assert!( + completed.is_ok(), + "TLS02_INVARIANT eight waiters cannot accumulate eight connect timeouts" + ); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(addr), + "TLS02_INVARIANT cancelled waiters release the lock registry" + ); + } + + #[test] + fn tls02_dial_registry_reclaims_unique_keys_and_preserves_live_lock_identity() { + for index in 0..1000 { + let key = format!("http://tls02-registry-{index}.invalid"); + let first = DialLock::for_key(&key); + let second = DialLock::for_key(&key); + assert!(Arc::ptr_eq(&first.mutex, &second.mutex), "TLS02_INVARIANT one live lock per cache key"); + drop(first); + assert!(DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key)); + drop(second); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key), + "TLS02_INVARIANT last owner removes its own weak entry" + ); + } + } + #[test] fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() { let mut cache = HashMap::new(); diff --git a/crates/protos/src/runtime_sources.rs b/crates/protos/src/runtime_sources.rs index 135e3a9f5..ac685d890 100644 --- a/crates/protos/src/runtime_sources.rs +++ b/crates/protos/src/runtime_sources.rs @@ -22,6 +22,10 @@ pub(crate) async fn outbound_tls_state() -> GlobalPublishedOutboundTlsState { load_global_outbound_tls_state().await } +pub(crate) fn outbound_tls_generation() -> u64 { + rustfs_tls_runtime::load_global_outbound_tls_generation().0 +} + pub(crate) fn record_stale_grpc_channel_tls_generation() { record_tls_consumer_stale_generation(PROTOS_GRPC_CHANNEL_TLS_CONSUMER); } diff --git a/crates/tls-runtime/Cargo.toml b/crates/tls-runtime/Cargo.toml index fb8920547..443be5ea0 100644 --- a/crates/tls-runtime/Cargo.toml +++ b/crates/tls-runtime/Cargo.toml @@ -46,6 +46,7 @@ sha2.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs", "rt", "sync", "time"] } tracing.workspace = true +x509-parser.workspace = true [dev-dependencies] rcgen.workspace = true diff --git a/crates/tls-runtime/src/certs.rs b/crates/tls-runtime/src/certs.rs index 20260768f..a79c8e9c0 100644 --- a/crates/tls-runtime/src/certs.rs +++ b/crates/tls-runtime/src/certs.rs @@ -429,6 +429,209 @@ pub fn load_all_certs_from_directory( Ok(cert_key_pairs) } +/// Strict runtime loader: a discovered identity must never disappear through +/// best-effort filtering. The diagnostic inspection API remains independent. +pub fn load_all_certs_from_directory_strict( + options: CertDirectoryLoadOptions, +) -> Result>, PrivateKeyDer<'static>)>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + options.validate()?; + let dir = &options.dir_path; + let mut pairs = HashMap::new(); + if let Some(pair) = read_cert_key_pair_strict(dir, &options.cert_filename, &options.key_filename, "default", true)? { + pairs.insert("default".to_string(), pair); + } + for entry in fs::read_dir(dir).map_err(|source| TlsRuntimeError::SourceIo { + path: dir.clone(), + source, + })? { + let entry = entry.map_err(|source| TlsRuntimeError::SourceIo { + path: dir.clone(), + source, + })?; + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_str().ok_or_else(|| TlsRuntimeError::InvalidCertificate { + name: "SNI directory".to_string(), + reason: "directory name is not UTF-8".to_string(), + })?; + if !is_discoverable_cert_domain_dir(name) { + continue; + } + let metadata = fs::metadata(&path).map_err(|source| TlsRuntimeError::SourceIo { + path: path.clone(), + source, + })?; + if !metadata.is_dir() { + continue; + } + if let Some(pair) = read_cert_key_pair_strict(&path, &options.cert_filename, &options.key_filename, name, true)? { + pairs.insert(name.to_string(), pair); + } + } + Ok(pairs) +} + +pub(crate) fn read_optional_tls_file(path: &Path) -> Result>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + match fs::symlink_metadata(path) { + Ok(_) => fs::read(path).map(Some).map_err(|source| TlsRuntimeError::SourceIo { + path: path.to_path_buf(), + source, + }), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + // A disappearing pinned generation is an error, not optional absence. + if let Some(parent) = path.parent() { + fs::read_dir(parent).map_err(|source| TlsRuntimeError::SourceIo { + path: parent.to_path_buf(), + source, + })?; + } + Ok(None) + } + Err(source) => Err(TlsRuntimeError::SourceIo { + path: path.to_path_buf(), + source, + }), + } +} + +pub(crate) fn parse_certificate_pem( + pem: &[u8], + name: &str, +) -> Result>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + let certs = CertificateDer::pem_slice_iter(pem) + .collect::, _>>() + .map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if certs.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "empty certificate PEM".to_string(), + }); + } + for cert in &certs { + let (remaining, _) = + x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if !remaining.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "trailing certificate DER".to_string(), + }); + } + } + Ok(certs) +} + +pub(crate) fn parse_private_key_pem(pem: &[u8], name: &str) -> Result, crate::error::TlsRuntimeError> { + PrivateKeyDer::from_pem_slice(pem).map_err(|error| crate::error::TlsRuntimeError::InvalidPrivateKey { + name: name.to_string(), + reason: error.to_string(), + }) +} + +pub(crate) fn read_cert_key_pair_strict( + dir: &Path, + cert_filename: &str, + key_filename: &str, + name: &str, + server: bool, +) -> Result>, PrivateKeyDer<'static>)>, crate::error::TlsRuntimeError> { + let cert = read_optional_tls_file(&dir.join(cert_filename))?; + let key = read_optional_tls_file(&dir.join(key_filename))?; + match (cert, key) { + (None, None) => Ok(None), + (Some(cert), Some(key)) => { + let certs = parse_certificate_pem(&cert, name)?; + let key = parse_private_key_pem(&key, name)?; + validate_certified_key(&certs, &key, name, server)?; + Ok(Some((certs, key))) + } + _ => Err(crate::error::TlsRuntimeError::IncompleteIdentity { name: name.to_string() }), + } +} + +/// Validate before publication, including the default certificate that bypasses +/// rustls' SNI name checks. +pub fn validate_certified_key( + certs: &[CertificateDer<'static>], + key: &PrivateKeyDer<'static>, + name: &str, + server: bool, +) -> Result { + use crate::error::TlsRuntimeError; + if certs.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "empty certificate chain".to_string(), + }); + } + for (index, cert) in certs.iter().enumerate() { + let (remaining, parsed) = + x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if !remaining.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "trailing certificate DER".to_string(), + }); + } + if index == 0 { + if !parsed.validity().is_valid() { + return Err(TlsRuntimeError::CertificateValidity { name: name.to_string() }); + } + if parsed + .extended_key_usage() + .map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })? + .is_some_and(|eku| { + if server { + !eku.value.server_auth + } else { + !eku.value.client_auth + } + }) + { + return Err(if server { + TlsRuntimeError::InvalidServerEku { name: name.to_string() } + } else { + TlsRuntimeError::InvalidClientEku { name: name.to_string() } + }); + } + } + } + let certified = CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &rustls::crypto::aws_lc_rs::default_provider()) + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + certified + .keys_match() + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + if server && name != "default" { + let mut sni = ResolvesServerCertUsingSni::new(); + sni.add(name, certified.clone()) + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + } + Ok(certified) +} + fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec>, PrivateKeyDer<'static>)> { let certs = load_certs(cert_path)?; let key = load_private_key(key_path)?; @@ -494,10 +697,7 @@ pub fn create_multi_cert_resolver( let mut default_cert = None; for (domain, (certs, key)) in cert_key_pairs { - let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?; - - let certified_key = CertifiedKey::new(certs, signing_key); + let certified_key = validate_certified_key(&certs, &key, &domain, true).map_err(io::Error::other)?; if domain == "default" { default_cert = Some(Arc::new(certified_key.clone())); } else { diff --git a/crates/tls-runtime/src/error.rs b/crates/tls-runtime/src/error.rs index ac7d50691..601624d6a 100644 --- a/crates/tls-runtime/src/error.rs +++ b/crates/tls-runtime/src/error.rs @@ -23,6 +23,38 @@ pub enum TlsRuntimeError { DirectoryNotFound { path: PathBuf }, #[error("TLS path is not a directory: {path}")] NotADirectory { path: PathBuf }, + #[error("TLS source I/O error at {path}: {source}")] + SourceIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("incomplete TLS certificate/private key pair for {name}")] + IncompleteIdentity { name: String }, + #[error("invalid TLS certificate material for {name}: {reason}")] + InvalidCertificate { name: String, reason: String }, + #[error("invalid TLS private key for {name}: {reason}")] + InvalidPrivateKey { name: String, reason: String }, + #[error("TLS certificate/private key validation failed for {name}: {source}")] + InvalidCertifiedKey { + name: String, + #[source] + source: rustls::Error, + }, + #[error("invalid TLS trust anchor in {name}: {source}")] + InvalidTrustAnchor { + name: String, + #[source] + source: rustls::Error, + }, + #[error("TLS certificate for {name} is outside its validity interval")] + CertificateValidity { name: String }, + #[error("TLS certificate for {name} does not permit server authentication")] + InvalidServerEku { name: String }, + #[error("TLS certificate for {name} does not permit client authentication")] + InvalidClientEku { name: String }, + #[error("TLS source loading task failed: {0}")] + SourceTaskJoin(#[from] tokio::task::JoinError), #[error("TLS material error: {0}")] Material(String), #[error("TLS publication error: {0}")] diff --git a/crates/tls-runtime/src/lib.rs b/crates/tls-runtime/src/lib.rs index 1d5bcf232..ae25ed86b 100644 --- a/crates/tls-runtime/src/lib.rs +++ b/crates/tls-runtime/src/lib.rs @@ -28,7 +28,8 @@ pub mod state; pub use certs::{ CertDirectoryLoadOptions, TlsCertPairInspection, TlsCertPairStatus, TlsDirectoryInspection, TlsDomainInspection, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver, inspect_cert_directory, - load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key, + load_all_certs_from_directory, load_all_certs_from_directory_strict, load_cert_bundle_der_bytes, load_certs, + load_private_key, validate_certified_key, }; pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions}; pub use coordinator::{TlsConsumer, TlsReloadCoordinator}; @@ -45,7 +46,7 @@ pub use outbound::{ load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state, }; pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop}; -pub use source::{TlsFileLayout, TlsSource, TlsSourceKind}; +pub use source::{TlsFileLayout, TlsSource, TlsSourceKind, TlsSourceSnapshot}; pub use state::OutboundOnlySnapshotArgs; pub use state::{TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeStatusSnapshot}; pub use state::{TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection}; @@ -111,6 +112,8 @@ mod tests { let coordinator = TlsReloadCoordinator::new(source.clone(), TlsReloadOptions::default()); let snapshot = TlsMaterialSnapshot { source, + server_client_ca_pem: None, + material_files: std::collections::BTreeSet::new(), server: Some(ServerTlsMaterial::MultiCert { cert_key_pairs: HashMap::new(), }), diff --git a/crates/tls-runtime/src/material.rs b/crates/tls-runtime/src/material.rs index f6b3fab99..5f8f200e5 100644 --- a/crates/tls-runtime/src/material.rs +++ b/crates/tls-runtime/src/material.rs @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory, load_certs, load_private_key}; +use crate::certs::{ + CertDirectoryLoadOptions, load_all_certs_from_directory_strict, parse_certificate_pem, parse_private_key_pem, + read_optional_tls_file, validate_certified_key, +}; use crate::error::TlsRuntimeError; use crate::fingerprint::TlsFingerprint; -use crate::source::TlsSource; +use crate::source::{TlsSource, TlsSourceSnapshot}; use rustfs_common::MtlsIdentityPem; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; -use std::collections::HashMap; -use std::io::Cursor; -use std::io::ErrorKind; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use std::collections::{BTreeSet, HashMap}; use std::path::Path; #[derive(Debug, Clone)] @@ -40,115 +41,196 @@ pub enum ServerTlsMaterial { }, } +impl ServerTlsMaterial { + /// Revalidate an accepted generation without reading the filesystem. + pub fn validate(&self) -> Result<(), TlsRuntimeError> { + match self { + Self::SingleCert { certs, key } => { + crate::certs::validate_certified_key(certs, key, "default", true)?; + } + Self::MultiCert { cert_key_pairs } => { + for (name, (certs, key)) in cert_key_pairs { + crate::certs::validate_certified_key(certs, key, name, true)?; + } + } + } + Ok(()) + } +} + +impl Clone for ServerTlsMaterial { + fn clone(&self) -> Self { + match self { + Self::SingleCert { certs, key } => Self::SingleCert { + certs: certs.clone(), + key: key.clone_key(), + }, + Self::MultiCert { cert_key_pairs } => Self::MultiCert { + cert_key_pairs: cert_key_pairs + .iter() + .map(|(name, (certs, key))| (name.clone(), (certs.clone(), key.clone_key()))) + .collect(), + }, + } + } +} + #[derive(Debug)] pub struct TlsMaterialSnapshot { + /// The resolved directory generation used by every consumer of this snapshot. pub source: TlsSource, pub server: Option, pub outbound: OutboundTlsMaterial, + pub server_client_ca_pem: Option>, + pub material_files: BTreeSet, pub fingerprint: TlsFingerprint, } impl TlsMaterialSnapshot { pub async fn load(source: &TlsSource) -> Result { - let base_dir = source.validate_directory()?.to_path_buf(); + let source = source.clone(); + tokio::task::spawn_blocking(move || Self::load_from_snapshot(&source.snapshot()?)).await? + } - let server = load_server_material(source, &base_dir)?; + /// Load all consumers' material from one previously pinned directory generation. + pub fn load_from_snapshot(snapshot: &TlsSourceSnapshot) -> Result { + Self::load_from_snapshot_with_server_directory(snapshot, None) + } - let public_ca_path = base_dir.join(&source.layout.public_ca_filename); - let client_ca_path = base_dir.join(&source.fallback_ca_filename); - let client_cert_path = base_dir.join(&source.layout.client_cert_filename); - let client_key_path = base_dir.join(&source.layout.client_key_filename); + /// Load a server-only publication without replacing the pinned trust or client identity source. + pub fn load_from_snapshot_with_server_directory( + snapshot: &TlsSourceSnapshot, + server_directory: Option<&Path>, + ) -> Result { + snapshot.validate_available()?; + let source = snapshot.resolved_source(); + let base_dir = snapshot.base_dir(); + let server = load_server_material(source, server_directory.unwrap_or(base_dir))?; + Self::load_from_snapshot_with_loaded_server(snapshot, server) + } - let public_ca_pem = tokio::fs::read(&public_ca_path).await.ok(); - let client_ca_pem = tokio::fs::read(&client_ca_path).await.ok(); - let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), client_ca_pem.as_deref()); + /// Validate an already decoded default server chain while reading trust and client identity from the pinned source. + /// Serving validity, EKU and key matching are checked here even when the publication ledger was verified earlier. + pub fn load_from_snapshot_with_server_identity( + snapshot: &TlsSourceSnapshot, + certificates: Vec>, + private_key_pem: &[u8], + ) -> Result { + snapshot.validate_available()?; + let key = parse_private_key_pem(private_key_pem, "default")?; + validate_certified_key(&certificates, &key, "default", true)?; + Self::load_from_snapshot_with_loaded_server( + snapshot, + Some(ServerTlsMaterial::SingleCert { + certs: certificates, + key, + }), + ) + } - let mtls_identity = match ( - tokio::fs::read(&client_cert_path).await.ok(), - tokio::fs::read(&client_key_path).await.ok(), - ) { - (Some(cert_pem), Some(key_pem)) => { - let mut cert_reader = Cursor::new(&cert_pem); - if CertificateDer::pem_reader_iter(&mut cert_reader).next().is_none() { - return Err(TlsRuntimeError::Material("no valid certificate in client cert PEM".to_string())); + fn load_from_snapshot_with_loaded_server( + snapshot: &TlsSourceSnapshot, + server: Option, + ) -> Result { + snapshot.validate_available()?; + let source = snapshot.resolved_source(); + let base_dir = snapshot.base_dir(); + let public_ca_pem = read_optional_tls_file(&base_dir.join(&source.layout.public_ca_filename))?; + let fallback_ca_pem = read_optional_tls_file(&base_dir.join(&source.fallback_ca_filename))?; + let client_ca_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_ca_filename))?; + let mut material_files = BTreeSet::new(); + for (name, present) in [ + (&source.layout.public_ca_filename, public_ca_pem.is_some()), + (&source.fallback_ca_filename, fallback_ca_pem.is_some()), + (&source.layout.client_ca_filename, client_ca_pem.is_some()), + ] { + if present { + material_files.insert(name.clone()); + } + } + for (name, pem) in [ + (&source.layout.public_ca_filename, &public_ca_pem), + (&source.fallback_ca_filename, &fallback_ca_pem), + (&source.layout.client_ca_filename, &client_ca_pem), + ] { + if let Some(pem) = pem { + let certs = parse_certificate_pem(pem, name)?; + let mut roots = rustls::RootCertStore::empty(); + for cert in certs { + roots.add(cert).map_err(|source| TlsRuntimeError::InvalidTrustAnchor { + name: name.clone(), + source, + })?; } - - let mut key_reader = Cursor::new(&key_pem); - PrivateKeyDer::from_pem_reader(&mut key_reader) - .map_err(|e| TlsRuntimeError::Material(format!("invalid client key PEM: {e}")))?; - + } + } + let server_client_ca_pem = client_ca_pem.or_else(|| fallback_ca_pem.clone()); + let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), fallback_ca_pem.as_deref()); + let client_cert_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_cert_filename))?; + let client_key_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_key_filename))?; + if client_cert_pem.is_some() { + material_files.insert(source.layout.client_cert_filename.clone()); + } + if client_key_pem.is_some() { + material_files.insert(source.layout.client_key_filename.clone()); + } + let mtls_identity = match (client_cert_pem, client_key_pem) { + (Some(cert_pem), Some(key_pem)) => { + let certs = parse_certificate_pem(&cert_pem, "client identity")?; + let key = parse_private_key_pem(&key_pem, "client identity")?; + validate_certified_key(&certs, &key, "client identity", false)?; Some(MtlsIdentityPem { cert_pem, key_pem }) } - _ => None, + (None, None) => None, + _ => { + return Err(TlsRuntimeError::IncompleteIdentity { + name: "client identity".to_string(), + }); + } }; - + snapshot.validate_available()?; let outbound = OutboundTlsMaterial { - root_ca_pem: root_ca_pem.clone(), + root_ca_pem, mtls_identity: mtls_identity.clone(), }; - - let server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + let mut server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + if let Some(pem) = &server_client_ca_pem { + let bytes = server_fingerprint_bytes.get_or_insert_with(Vec::new); + bytes.extend_from_slice(b"\0server-client-ca\0"); + bytes.extend_from_slice(pem); + } let fingerprint = TlsFingerprint::from_optional_bytes( server_fingerprint_bytes.as_deref(), public_ca_pem.as_deref(), - client_ca_pem.as_deref(), + fallback_ca_pem.as_deref(), mtls_identity.as_ref().map(|identity| identity.cert_pem.as_slice()), mtls_identity.as_ref().map(|identity| identity.key_pem.as_slice()), ); - Ok(Self { - source: source.clone(), + source: snapshot.resolved_source().clone(), server, outbound, + server_client_ca_pem, + material_files, fingerprint, }) } } fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result, TlsRuntimeError> { - let root_cert = base_dir.join(&source.layout.server_cert_filename); - let root_key = base_dir.join(&source.layout.server_key_filename); - - let has_root_pair = root_cert.exists() && root_key.exists(); - let cert_key_pairs = load_all_certs_from_directory( + let mut cert_key_pairs = load_all_certs_from_directory_strict( CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename) .build(), - ); - - match cert_key_pairs { - Ok(cert_key_pairs) if cert_key_pairs.len() > 1 || cert_key_pairs.keys().any(|key| key != "default") => { - Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) - } - Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => { - if let Some((certs, key)) = cert_key_pairs.get("default") { - return Ok(Some(ServerTlsMaterial::SingleCert { - certs: certs.clone(), - key: key.clone_key(), - })); - } - Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) - } - Ok(_) => Ok(None), - Err(_err) if has_root_pair => { - let root_cert_path = path_to_utf8_str(&root_cert, "root TLS certificate")?; - let root_key_path = path_to_utf8_str(&root_key, "root TLS private key")?; - let certs = load_certs(root_cert_path) - .map_err(|e| TlsRuntimeError::Material(format!("load root TLS certificate {}: {e}", root_cert.display())))?; - let key = load_private_key(root_key_path) - .map_err(|e| TlsRuntimeError::Material(format!("load root TLS private key {}: {e}", root_key.display())))?; - Ok(Some(ServerTlsMaterial::SingleCert { certs, key })) - } - Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), - Err(err) => Err(TlsRuntimeError::Material(format!( - "discover server TLS certificates under '{}': {err}", - base_dir.display() - ))), + )?; + if cert_key_pairs.is_empty() { + return Ok(None); } -} - -fn path_to_utf8_str<'a>(path: &'a Path, description: &str) -> Result<&'a str, TlsRuntimeError> { - path.to_str() - .ok_or_else(|| TlsRuntimeError::Material(format!("{description} path '{}' is not valid UTF-8", path.display()))) + if cert_key_pairs.len() == 1 { + if let Some((certs, key)) = cert_key_pairs.remove("default") { + return Ok(Some(ServerTlsMaterial::SingleCert { certs, key })); + } + } + Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) } fn combine_optional_pem(primary: Option<&[u8]>, fallback: Option<&[u8]>) -> Vec { @@ -198,3 +280,425 @@ pub(crate) fn server_material_fingerprint(material: &ServerTlsMaterial) -> TlsFi let bytes = serialize_server_material_for_fingerprint(material); TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None) } + +#[cfg(test)] +mod tests { + use super::*; + use rcgen::{CertificateParams, ExtendedKeyUsagePurpose, KeyPair}; + use std::fs; + use tempfile::TempDir; + + fn write_pair(dir: &Path, domain: &str) { + let pair = rcgen::generate_simple_self_signed(vec![domain.to_string()]).expect("generate certificate"); + fs::write(dir.join("rustfs_cert.pem"), pair.cert.pem()).expect("write certificate"); + fs::write(dir.join("rustfs_key.pem"), pair.signing_key.serialize_pem()).expect("write private key"); + } + + #[test] + fn provided_server_identity_does_not_reread_server_files_and_preserves_trust() { + let base = TempDir::new().expect("base source"); + let serving = rcgen::generate_simple_self_signed(vec!["example.com".into()]).expect("server identity"); + let trust = rcgen::generate_simple_self_signed(vec!["trust.example".into()]).expect("trust source"); + fs::write(base.path().join("rustfs_cert.pem"), "unusable server certificate").expect("poison base certificate"); + fs::write(base.path().join("rustfs_key.pem"), "unusable server key").expect("poison base key"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), trust.cert.pem()).expect("trust file"); + let source = TlsSource::from_directory(base.path()).snapshot().expect("source"); + let loaded = TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + serving.signing_key.serialize_pem().as_bytes(), + ) + .expect("provided identity"); + assert_eq!(loaded.outbound.root_ca_pem, trust.cert.pem().as_bytes()); + assert!( + matches!(loaded.server, Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == vec![serving.cert.der().clone()]) + ); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), "corrupt trust").expect("corrupt independent trust"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + serving.signing_key.serialize_pem().as_bytes() + ) + .is_err(), + "providing server material must not bypass trust validation" + ); + } + + #[test] + fn provided_server_identity_preserves_key_time_and_eku_validation() { + let base = TempDir::new().expect("base source"); + let source = TlsSource::from_directory(base.path()).snapshot().expect("source"); + let serving = rcgen::generate_simple_self_signed(vec!["example.com".into()]).expect("server identity"); + let wrong_key = KeyPair::generate().expect("wrong key"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "key matching remains mandatory" + ); + let mut expired = CertificateParams::new(vec!["example.com".into()]).expect("parameters"); + expired.not_before = rcgen::date_time_ymd(2000, 1, 1); + expired.not_after = rcgen::date_time_ymd(2001, 1, 1); + let cert = expired.self_signed(&wrong_key).expect("expired certificate"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "serving validity remains mandatory" + ); + let mut client = CertificateParams::new(vec!["example.com".into()]).expect("parameters"); + client.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let cert = client.self_signed(&wrong_key).expect("client certificate"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "server EKU remains mandatory" + ); + } + + #[tokio::test] + async fn server_overlay_survives_expired_bootstrap_without_replacing_trust() { + let base = TempDir::new().expect("base source"); + let overlay = TempDir::new().expect("server overlay"); + let key = KeyPair::generate().expect("bootstrap key"); + let mut params = CertificateParams::new(vec!["example.com".into()]).expect("bootstrap params"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let expired = params.self_signed(&key).expect("expired bootstrap"); + fs::write(base.path().join("rustfs_cert.pem"), expired.pem()).expect("bootstrap certificate"); + fs::write(base.path().join("rustfs_key.pem"), key.serialize_pem()).expect("bootstrap key"); + let ca = rcgen::generate_simple_self_signed(vec!["trust.example".into()]).expect("trust source"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), ca.cert.pem()).expect("trust file"); + write_pair(overlay.path(), "example.com"); + let source = TlsSource::from_directory(base.path()); + assert!( + TlsMaterialSnapshot::load(&source).await.is_err(), + "the expired base alone must remain invalid" + ); + let loaded = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &source.snapshot().expect("pinned base"), + Some(overlay.path()), + ) + .expect("valid overlay"); + assert_eq!(loaded.outbound.root_ca_pem, ca.cert.pem().as_bytes()); + assert!(matches!(loaded.server, Some(ServerTlsMaterial::SingleCert { .. }))); + } + + #[tokio::test] + async fn snapshot_rejects_mismatched_default_private_key() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let other = KeyPair::generate().expect("generate other key"); + fs::write(dir.path().join("rustfs_key.pem"), other.serialize_pem()).expect("replace key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_partial_sni_pair_even_with_valid_default() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create SNI directory"); + fs::write(domain.join("rustfs_cert.pem"), "invalid certificate").expect("write partial SNI pair"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_invalid_existing_ca() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join(&source.layout.public_ca_filename), "invalid CA").expect("write invalid CA"); + assert!(TlsMaterialSnapshot::load(&source).await.is_err()); + } + + #[tokio::test] + async fn snapshot_rejects_incomplete_client_identity() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join(&source.layout.client_cert_filename), "invalid client certificate").expect("write client cert"); + assert!(TlsMaterialSnapshot::load(&source).await.is_err()); + } + + #[tokio::test] + async fn snapshot_rejects_expired_server_certificate() { + let dir = TempDir::new().expect("create temp directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let cert = params.self_signed(&key).expect("generate expired certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_client_only_server_eku() { + let dir = TempDir::new().expect("create temp directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let cert = params.self_signed(&key).expect("generate client-only certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_server_only_client_eku() { + let dir = TempDir::new().expect("client identity directory"); + write_pair(dir.path(), "server.example"); + let key = KeyPair::generate().expect("client key"); + let mut params = CertificateParams::new(vec!["client.example".to_string()]).expect("client params"); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let cert = params.self_signed(&key).expect("server-only certificate"); + fs::write(dir.path().join("client_cert.pem"), cert.pem()).expect("client certificate"); + fs::write(dir.path().join("client_key.pem"), key.serialize_pem()).expect("client key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::InvalidClientEku { .. }) + )); + } + #[tokio::test] + async fn snapshot_rejects_future_server_certificate() { + let dir = TempDir::new().expect("create directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.not_before = rcgen::date_time_ymd(2100, 1, 1); + params.not_after = rcgen::date_time_ymd(2101, 1, 1); + let cert = params.self_signed(&key).expect("generate future certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::CertificateValidity { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_wrong_sni_certificate_name() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("other.example.com"); + fs::create_dir(&domain).expect("create domain directory"); + write_pair(&domain, "wrong.example.com"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_malformed_complete_sni_pair() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create domain directory"); + write_pair(&domain, "sni.example.com"); + fs::write(domain.join("rustfs_cert.pem"), "invalid certificate").expect("corrupt domain certificate"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_cert_only_root() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + fs::remove_file(dir.path().join("rustfs_key.pem")).expect("remove key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::IncompleteIdentity { .. }) + )); + } + + #[tokio::test] + async fn snapshot_fingerprint_is_stable_and_certificate_only_renewal_changes_it() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + let initial = TlsMaterialSnapshot::load(&source) + .await + .expect("load valid initial generation"); + let unchanged = TlsMaterialSnapshot::load(&source).await.expect("load unchanged generation"); + assert_eq!(initial.fingerprint, unchanged.fingerprint); + let key = KeyPair::from_pem(&fs::read_to_string(dir.path().join("rustfs_key.pem")).expect("read key")) + .expect("parse existing key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.serial_number = Some(123_u64.into()); + let renewed = params.self_signed(&key).expect("renew certificate with existing key"); + fs::write(dir.path().join("rustfs_cert.pem"), renewed.pem()).expect("write renewed certificate"); + let changed = TlsMaterialSnapshot::load(&source).await.expect("load renewed generation"); + assert_ne!(initial.fingerprint, changed.fingerprint); + } + + #[tokio::test] + async fn snapshot_server_client_ca_rotation_changes_fingerprint() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + let first_ca = rcgen::generate_simple_self_signed(vec!["first-ca".to_string()]).expect("first CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), first_ca.cert.pem()).expect("write first CA"); + let initial = TlsMaterialSnapshot::load(&source).await.expect("load first CA"); + let second_ca = rcgen::generate_simple_self_signed(vec!["second-ca".to_string()]).expect("second CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), second_ca.cert.pem()).expect("write second CA"); + let changed = TlsMaterialSnapshot::load(&source).await.expect("load second CA"); + assert_ne!(initial.fingerprint, changed.fingerprint); + assert_eq!(changed.server_client_ca_pem.as_deref(), Some(second_ca.cert.pem().as_bytes())); + } + + #[cfg(unix)] + #[test] + fn snapshot_kubernetes_switch_loads_all_material_from_pinned_generation() { + use std::os::unix::fs::symlink; + let dir = TempDir::new().expect("create projection"); + let source = TlsSource::from_directory(dir.path()); + for generation in ["..a", "..b"] { + let path = dir.path().join(generation); + fs::create_dir(&path).expect("create generation"); + write_pair(&path, "example.com"); + let cert = fs::read(path.join(&source.layout.server_cert_filename)).expect("read generated certificate"); + let key = fs::read(path.join(&source.layout.server_key_filename)).expect("read generated key"); + for filename in [ + &source.layout.public_ca_filename, + &source.fallback_ca_filename, + &source.layout.client_ca_filename, + &source.layout.client_cert_filename, + ] { + fs::write(path.join(filename), &cert).expect("write generation CA/client certificate"); + } + fs::write(path.join(&source.layout.client_key_filename), key).expect("write generation client key"); + } + symlink("..a", dir.path().join("..data")).expect("link initial generation"); + for filename in [ + &source.layout.server_cert_filename, + &source.layout.server_key_filename, + &source.layout.public_ca_filename, + &source.fallback_ca_filename, + &source.layout.client_ca_filename, + &source.layout.client_cert_filename, + &source.layout.client_key_filename, + ] { + symlink(format!("..data/{filename}"), dir.path().join(filename)).expect("link projected file"); + } + let pinned = source.snapshot().expect("pin initial generation"); + let expected = TlsMaterialSnapshot::load_from_snapshot(&pinned).expect("load initial generation"); + symlink("..b", dir.path().join("..next")).expect("link next generation"); + fs::rename(dir.path().join("..next"), dir.path().join("..data")).expect("switch projection"); + let actual = TlsMaterialSnapshot::load_from_snapshot(&pinned).expect("load pinned generation after switch"); + assert_eq!(actual.fingerprint, expected.fingerprint); + assert_eq!(actual.source.base_dir, pinned.base_dir()); + assert_eq!(actual.outbound.root_ca_pem, expected.outbound.root_ca_pem); + assert_eq!(actual.server_client_ca_pem, expected.server_client_ca_pem); + assert_eq!( + actual.outbound.mtls_identity.expect("client identity").key_pem, + expected.outbound.mtls_identity.expect("expected identity").key_pem + ); + let current = TlsMaterialSnapshot::load_from_snapshot(&source.snapshot().expect("pin new generation")) + .expect("load new generation"); + assert_ne!(actual.fingerprint, current.fingerprint); + } + + #[cfg(unix)] + #[test] + fn snapshot_rejects_disappeared_projection_source_and_dangling_optional_ca() { + use std::os::unix::fs::symlink; + let dir = TempDir::new().expect("create directory"); + let source = TlsSource::from_directory(dir.path()); + write_pair(dir.path(), "example.com"); + symlink("missing-ca", dir.path().join(&source.layout.public_ca_filename)).expect("link missing CA"); + let pinned = source.snapshot().expect("pin directory"); + assert!(matches!( + TlsMaterialSnapshot::load_from_snapshot(&pinned), + Err(TlsRuntimeError::SourceIo { .. }) + )); + fs::remove_file(dir.path().join(&source.layout.public_ca_filename)).expect("remove CA link"); + fs::remove_dir_all(dir.path()).expect("remove pinned source"); + assert!(matches!( + TlsMaterialSnapshot::load_from_snapshot(&pinned), + Err(TlsRuntimeError::SourceIo { .. }) + )); + } + #[tokio::test] + async fn snapshot_rejects_mismatched_sni_key_and_malformed_chain_member() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create SNI directory"); + write_pair(&domain, "sni.example.com"); + let source = TlsSource::from_directory(dir.path()); + let valid = TlsMaterialSnapshot::load(&source) + .await + .expect("load valid multi certificate material"); + assert!(matches!(valid.server, Some(ServerTlsMaterial::MultiCert { ref cert_key_pairs }) if cert_key_pairs.len() == 2)); + fs::write( + domain.join("rustfs_key.pem"), + KeyPair::generate().expect("generate different key").serialize_pem(), + ) + .expect("replace SNI key"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertifiedKey { .. }) + )); + write_pair(&domain, "sni.example.com"); + let cert_path = dir.path().join("rustfs_cert.pem"); + let mut cert = fs::read_to_string(&cert_path).expect("read leaf certificate"); + cert.push_str("-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n"); + fs::write(cert_path, cert).expect("append malformed chain member"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_invalid_private_key_pem_and_invalid_primary_client_ca() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join("rustfs_key.pem"), "invalid key").expect("corrupt key"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidPrivateKey { .. }) + )); + write_pair(dir.path(), "example.com"); + let valid_ca = fs::read(dir.path().join("rustfs_cert.pem")).expect("read valid certificate"); + fs::write(dir.path().join(&source.fallback_ca_filename), valid_ca).expect("write valid fallback CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), "invalid primary CA").expect("write invalid primary CA"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } +} diff --git a/crates/tls-runtime/src/outbound.rs b/crates/tls-runtime/src/outbound.rs index 11c9c5916..fac3a65cb 100644 --- a/crates/tls-runtime/src/outbound.rs +++ b/crates/tls-runtime/src/outbound.rs @@ -16,8 +16,8 @@ use crate::material::OutboundTlsMaterial; use crate::metrics::record_outbound_tls_publication; use crate::state::TlsGeneration; use rustfs_common::{ - MtlsIdentityPem, clear_global_root_cert, get_global_mtls_identity, get_global_outbound_tls_generation, get_global_root_cert, - set_global_mtls_identity, set_global_outbound_tls_generation, set_global_root_cert, + MtlsIdentityPem, OutboundTlsSnapshot, get_global_outbound_tls_generation, get_global_outbound_tls_snapshot, + publish_global_outbound_tls_snapshot, }; #[derive(Debug, Clone)] @@ -35,21 +35,21 @@ pub struct GlobalOutboundTlsStateSummary { } pub async fn publish_global_outbound_tls_state(generation: TlsGeneration, material: &OutboundTlsMaterial) { - if !material.root_ca_pem.is_empty() { - set_global_root_cert(material.root_ca_pem.clone()).await; - } else { - clear_global_root_cert().await; - } - set_global_mtls_identity(material.mtls_identity.clone()).await; - set_global_outbound_tls_generation(generation.0); - record_outbound_tls_publication(generation.0, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); + let accepted = publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: generation.0, + root_ca_pem: (!material.root_ca_pem.is_empty()).then(|| material.root_ca_pem.clone()), + mtls_identity: material.mtls_identity.clone(), + }) + .await; + record_outbound_tls_publication(accepted, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); } pub async fn load_global_outbound_tls_state() -> GlobalPublishedOutboundTlsState { + let snapshot = get_global_outbound_tls_snapshot(); GlobalPublishedOutboundTlsState { - generation: TlsGeneration(get_global_outbound_tls_generation()), - root_ca_pem: get_global_root_cert().await, - mtls_identity: get_global_mtls_identity().await, + generation: TlsGeneration(snapshot.generation), + root_ca_pem: snapshot.root_ca_pem.clone(), + mtls_identity: snapshot.mtls_identity.clone(), } } diff --git a/crates/tls-runtime/src/source.rs b/crates/tls-runtime/src/source.rs index dee0b0277..a35b68001 100644 --- a/crates/tls-runtime/src/source.rs +++ b/crates/tls-runtime/src/source.rs @@ -92,3 +92,125 @@ impl TlsSource { Ok(self.base_dir.as_path()) } } + +/// A directory generation resolved once before loading any TLS material. +/// +/// Projected Kubernetes volumes are read through their physical `..data` +/// directory. Ordinary directory and certificate-file symlinks remain supported +/// for layouts such as Certbot. Writers must keep pinned generations immutable. +#[derive(Debug, Clone)] +pub struct TlsSourceSnapshot { + source: TlsSource, + resolved_source: TlsSource, +} + +impl TlsSource { + pub fn snapshot(&self) -> Result { + self.validate_directory()?; + let mut resolved_source = self.clone(); + let root = std::fs::canonicalize(&self.base_dir).map_err(|source| TlsRuntimeError::SourceIo { + path: self.base_dir.clone(), + source, + })?; + let projection = root.join("..data"); + resolved_source.base_dir = match std::fs::symlink_metadata(&projection) { + Ok(metadata) if metadata.file_type().is_symlink() => { + std::fs::canonicalize(&projection).map_err(|source| TlsRuntimeError::SourceIo { + path: projection, + source, + })? + } + Ok(_) => root, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => root, + Err(source) => { + return Err(TlsRuntimeError::SourceIo { + path: projection, + source, + }); + } + }; + resolved_source.validate_directory()?; + Ok(TlsSourceSnapshot { + source: self.clone(), + resolved_source, + }) + } +} + +impl TlsSourceSnapshot { + pub fn source(&self) -> &TlsSource { + &self.source + } + + pub fn resolved_source(&self) -> &TlsSource { + &self.resolved_source + } + + pub fn base_dir(&self) -> &Path { + &self.resolved_source.base_dir + } + + /// Missing optional files are allowed only while the pinned source exists. + pub(crate) fn validate_available(&self) -> Result<(), TlsRuntimeError> { + std::fs::read_dir(self.base_dir()).map_err(|source| TlsRuntimeError::SourceIo { + path: self.base_dir().to_path_buf(), + source, + })?; + Ok(()) + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::fs; + use std::os::unix::fs::symlink; + use tempfile::TempDir; + + #[test] + fn source_snapshot_pins_kubernetes_projection_generation() { + let dir = TempDir::new().expect("create projection directory"); + for generation in ["..generation-a", "..generation-b"] { + fs::create_dir(dir.path().join(generation)).expect("create generation"); + fs::write(dir.path().join(generation).join("marker"), generation).expect("write generation marker"); + } + symlink("..generation-a", dir.path().join("..data")).expect("link initial generation"); + symlink("..data/marker", dir.path().join("marker")).expect("link projected file"); + let pinned = TlsSource::from_directory(dir.path()).snapshot().expect("pin projection"); + symlink("..generation-b", dir.path().join("..next-data")).expect("link next generation"); + fs::rename(dir.path().join("..next-data"), dir.path().join("..data")).expect("swap projection"); + assert_eq!( + fs::read_to_string(pinned.base_dir().join("marker")).expect("read pinned file"), + "..generation-a" + ); + assert_eq!( + fs::read_to_string(dir.path().join("marker")).expect("read current projection"), + "..generation-b" + ); + } + + #[test] + fn source_snapshot_pins_tls_path_directory_symlink() { + let dir = TempDir::new().expect("create TLS directory"); + fs::create_dir(dir.path().join("a")).expect("create generation A"); + fs::create_dir(dir.path().join("b")).expect("create generation B"); + symlink("a", dir.path().join("live")).expect("link generation A"); + let pinned = TlsSource::from_directory(dir.path().join("live")) + .snapshot() + .expect("pin TLS source"); + fs::remove_file(dir.path().join("live")).expect("unlink source"); + symlink("b", dir.path().join("live")).expect("link generation B"); + assert_eq!(pinned.base_dir(), fs::canonicalize(dir.path().join("a")).expect("resolve generation A")); + } + + #[test] + fn source_snapshot_rejects_disappeared_generation() { + let dir = TempDir::new().expect("create TLS directory"); + let generation = dir.path().join("..generation"); + fs::create_dir(&generation).expect("create generation"); + symlink("..generation", dir.path().join("..data")).expect("link generation"); + let pinned = TlsSource::from_directory(dir.path()).snapshot().expect("pin TLS source"); + fs::remove_dir(&generation).expect("remove pinned generation"); + assert!(matches!(pinned.validate_available(), Err(TlsRuntimeError::SourceIo { .. }))); + } +} diff --git a/docs/README.md b/docs/README.md index 89549fcd2..2b89a015d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,8 @@ collection: ## Operations +- [External certificate issuance and TLS rotation](operations/tls-certificate-rotation.md) — publication, native reload, trust separation and failure handling. + - [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures. - [Logging governance](operations/logging-governance.md) — broad event audits, migrations, and guardrail changes. diff --git a/docs/operations/tls-certificate-rotation.md b/docs/operations/tls-certificate-rotation.md new file mode 100644 index 000000000..750f17c6d --- /dev/null +++ b/docs/operations/tls-certificate-rotation.md @@ -0,0 +1,204 @@ +# External certificate issuance and TLS rotation + +Use an independent certificate controller to obtain and renew certificates. +RustFS consumes certificate material; it does not register ACME accounts or +change DNS records. The independent +[rustfs-cert-controller](https://github.com/houseme/rustfs-cert-controller) +owns issuance, renewal scheduling and publication. `plugin-runner` starts its +container; it does not own certificate schedules or CA credentials. Progress +and deployment acceptance are tracked in +[backlog #2806](https://github.com/rustfs/backlog/issues/2806). + +## Deployment boundaries + +For a distributed Kubernetes deployment, prefer public TLS at an +Ingress/Gateway and private TLS/mTLS between the ingress and RustFS. Public +CA certificates cannot cover internal names such as `*.svc.cluster.local`; +DNS-01 can cover names under a controlled public domain even when the service +is private. A listener +that requires mTLS also requires client certificates from browsers and S3 +clients; selecting a different SNI certificate does not change that policy. + +Native server TLS can consume an externally managed `kubernetes.io/tls` +Secret with only `tls.crt` and `tls.key`: + +```yaml +mode: + distributed: + enabled: false + standalone: + enabled: true +ingress: + enabled: false +tls: + enabled: true + existingSecret: rustfs-serving-tls + reloadInterval: 30 +``` + +Create and renew that Secret using cert-manager or an independently managed +publisher. Configure the issuing resource with the actual server DNS/IP SANs +and explicit private-key rotation. The chart does not install cert-manager, +create an ACME account or grant DNS permissions. For distributed native TLS, +the certificate must also cover the configured peer hostnames; explicitly +configure independent outbound trust when a private server CA is used. + +`tls.trustSecret` optionally names a separate Secret with an explicitly chosen +`ca.crt`, mounted as outbound `public.crt`. It does not enable mTLS or add a +public issuing CA to client authentication trust. Do not write a fabricated +`ca.crt` into the Secret managed by an ACME issuer. + +`tls.enabled` and `mtls.enabled` are mutually exclusive. Existing mTLS mode +keeps its separate server/client certificate Secrets and projects their +contents into one read-only directory. The chart enables certificate reload +for either mode and disables TLS key logging. Explicit container environment +variables can override these settings. + +## Material layout and publication + +`RUSTFS_TLS_PATH` contains `rustfs_cert.pem` and `rustfs_key.pem`. SNI +identities can reside in visible domain subdirectories with the same two +filenames. Outbound trust uses `public.crt` and `ca.crt`; inbound mTLS client +trust prefers `client_ca.crt`, falling back to `ca.crt` only at initial load. +The default outbound client identity is `client_cert.pem`/`client_key.pem`. + +Runtime loading validates existing PEM/DER, key pairing, leaf validity and +explicit authentication EKUs. Each named SNI identity must match its domain. +Chain structure validation does not replace issuer/trust-policy verification +by the issuing controller and client PKI verifier. + +Kubernetes mounts must project the entire directory; single-file `subPath` +mounts do not receive Secret updates. RustFS pins one physical `..data` +generation before loading its material. Separate controllers may update the +serving identity and trust independently, so private CA migration still needs +an old-plus-new trust overlap. + +For VMs, publish immutable version directories and atomically change the +directory pointer after syncing the complete candidate and its containing +directories. Keep current/accepted/in-use versions during cleanup. Certbot +file symlinks remain compatible, but independent file replacements do not +provide an atomic directory generation; key mismatch is rejected, and a +publisher should use a complete immutable bundle for coherent publication. + +`RUSTFS_MTLS_CLIENT_CERT`/`RUSTFS_MTLS_CLIENT_KEY` overrides are validated as +one identity. Explicit missing, malformed, mismatched, expired or unsuitable +override material rejects the update; it does not silently select another +identity. Keep default identity files valid when they are present as well. + +## Controller publication API + +For a VM or Docker installation with one default serving certificate, enable +reload and configure `RUSTFS_TLS_PUBLISH_DIR` to a dedicated persistent directory +owned by the RustFS process with mode `0700`. Keep `RUSTFS_TLS_PATH` configured +for the initial serving identity and independent trust/client material. The +publication directory must be separate from the original TLS source. A valid +initial certificate and trusted management credentials must be provisioned +before a controller can connect to the HTTPS API. That bootstrap identity must +cover the names the configured issuer will continue to renew; publication +preserves existing SAN coverage. + +The controller signs requests using the existing S3 SigV4 authentication: + +- `GET /rustfs/admin/v3/tls/certificate-state` requires + `admin:TLSCertificateStatus`. +- `POST /rustfs/admin/v3/tls/certificate` requires + `admin:TLSCertificatePublish` and a connection terminated by RustFS TLS. + +Grant only these two actions to the controller identity. Read permissions, +general configuration permissions and forwarded HTTPS headers do not authorize +private-key publication. Keep API requests and credentials out of access-body +logs. The API does not add MinIO-compatible aliases. + +The `rustfs.tls-publish/v1alpha1` request fixes `resource_id` to `server-default` +and carries an `issuance_id`, the captured `expected_target_revision`, +`certificate_pem` and `private_key_pem`. Read the target token before issuance; +retain it with that immutable candidate. Repeating the same issuance and payload +recovers a lost receipt. A conflict stops that candidate; fetching a new token +and replaying the old candidate would defeat this concurrency protection. +An HTTP 503 or uncertain network result is retryable with the same issuance, +payload and token; HTTP 409 indicates a revision or payload conflict. + +RustFS validates the complete candidate before publishing an immutable leaf +generation and syncing its atomic pointer. It does not copy or overwrite CA +trust or client credentials. This mode rejects projected Kubernetes sources, +multiple SNI identities and configurations that trust the serving leaf as a CA. +Use the existing external-file/cert-manager integration for those deployments. + +The state reports the visible target revision and public leaf fingerprint. A +successful POST receipt proves the publication was synced; a GET during an +uncertain write does not replace that receipt. Retry the identical POST to +confirm acceptance. `observed_certificate_fingerprint` is present only when +this process's serving reload group has the exact current certificate chain; +matching the leaf alone is insufficient. A durable receipt can precede that +observation. Check convergence and a fresh TLS handshake on every relevant +endpoint. One process's response does not prove another node has converged. + +Use a stable local publication volume with one owner. Captured-token CAS and +local process locking do not implement HA leadership, restore ordering or PITR +protection. Do not remove active generation directories; automated generation +cleanup is not part of this interface. + +## Activation and failure handling + +Native `RUSTFS_TLS_RELOAD_ENABLE` defaults to false. Set it to `1` outside the +chart. `RUSTFS_TLS_RELOAD_INTERVAL` defaults to 30 seconds and is clamped to a +minimum of 5 seconds. + +Listeners sharing the same configured TLS directory use one reload group. +The group prepares TCP and enabled HTTP/3 configurations from the same +snapshot before swapping them. A late listener first joins the accepted +configuration, even if the current files have become invalid. TCP, QUIC and +separate processes still have a short activation window; there is no global +atomic certificate switch. +TCP and QUIC derive their protocol settings from one prepared certificate +resolver and client verifier for that generation. Publication state and retries +load each immutable generation once per operation; the next operation validates +its disk contents again. Reload uses the verified serving identity with the +independently pinned trust/client source, without reopening its certificate and +key files. Candidate staging still performs its separate disk validation. + +Unchanged material is skipped. A serving certificate renewal does not advance +the outbound generation unless outbound trust or identity actually changes. +Outbound state is published as one immutable trust/identity/generation +snapshot. Control and bulk RPC cache acquisition checks that generation; +existing in-flight calls retain their channels. + +Reload rejects the disappearance of accepted CA/identity source components, +SNI entries or accepted DNS/IP SAN coverage. Removing or renaming an identity +or trust source requires an explicit configuration change and restart; +ordinary renewal must preserve those requirements. Rejected updates retain +the last accepted in-memory configuration. New handshakes reject certificates +outside their validity interval. Normal renewal does not terminate existing +uploads, downloads or sessions. + +The chart uses TCP listener liveness for TLS deployments so certificate expiry +does not trigger a restart loop. Readiness continues to check the TLS service. +This liveness probe detects listener availability, not complete application +health. Keep ingress traffic admission separate from internal peer routing +and storage quorum. Do not restart every node because a CA is unavailable. + +There is no automatic persisted private-key fallback in Kubernetes. On +restart, invalid explicitly configured TLS material prevents startup. Key +compromise, explicit certificate revocation, session invalidation, per-purpose +profile coordination and deployment-boundary fencing require the separate +operator/control-plane workflow; ordinary reload does not implement them. + +## Verification + +Check a fresh TLS connection to each endpoint and compare the leaf serial, +public certificate fingerprint, SANs and expiry. Include HTTP/2, enabled +HTTP/3, Console and internal RPC endpoints. Do not infer served certificates +from Secret contents or an outbound generation gauge alone. + +Focused local checks: + +```sh +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-tls-runtime +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-protos --lib +bash scripts/test_helm_tls_reload.sh +``` + +Production acceptance also requires real Kubernetes projection, CA/DNS +issuance and renewal, outage/expiry, long-transfer continuity, private CA +overlap and distributed convergence tests. These are separate from protocol +and local loader tests. diff --git a/helm/rustfs/templates/_helpers.tpl b/helm/rustfs/templates/_helpers.tpl index 126e07e4f..4c3cac02a 100644 --- a/helm/rustfs/templates/_helpers.tpl +++ b/helm/rustfs/templates/_helpers.tpl @@ -154,6 +154,10 @@ Merges (in order of increasing precedence): }} {{- $annotations = merge $annotations $mtls }} {{- end }} +{{- $tls := .Values.tls | default dict -}} +{{- if and ($tls.enabled | default false) (eq .Values.ingress.className "traefik") -}} +{{- $_ := set $annotations "traefik.ingress.kubernetes.io/service.serversscheme" "https" -}} +{{- end -}} {{- if $annotations }} {{- toYaml $annotations }} {{- end }} @@ -188,6 +192,10 @@ Merges (in order of increasing precedence): {{- end }} {{- $annotations = merge $annotations (default (dict) .Values.ingress.customAnnotations) }} {{- $annotations = merge $annotations (default (dict) .Values.ingress.annotations) }} +{{- $tls := .Values.tls | default dict -}} +{{- if and ($tls.enabled | default false) (eq .Values.ingress.className "nginx") -}} +{{- $_ := set $annotations "nginx.ingress.kubernetes.io/backend-protocol" "HTTPS" -}} +{{- end -}} {{- if $annotations }} {{- toYaml $annotations }} {{- end }} @@ -285,7 +293,8 @@ RUSTFS_VOLUMES on spaces, one pool per expression). */}} {{- define "rustfs.volumes" -}} {{- $protocol := "http" -}} -{{- if .Values.mtls.enabled -}} +{{- $tls := .Values.tls | default dict -}} +{{- if or .Values.mtls.enabled ($tls.enabled | default false) -}} {{- $protocol = "https" -}} {{- end -}} {{- $headless := printf "%s-headless" (include "rustfs.fullname" .) -}} @@ -347,17 +356,17 @@ Render liveness and readiness probe for http and https */}} {{- define "rustfs.probes" -}} +{{- $tls := .Values.tls | default dict -}} {{- if .Values.livenessProbe.enabled }} livenessProbe: - {{- if .Values.mtls.enabled }} - exec: - command: -{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health") | nindent 6 }} + {{- if or .Values.mtls.enabled ($tls.enabled | default false) }} + tcpSocket: + port: {{ .Values.service.endpoint.port | default 9000 }} {{- else }} httpGet: path: /health port: {{ .Values.service.endpoint.port | default 9000 }} - scheme: {{ if .Values.mtls.enabled }}HTTPS{{ else }}HTTP{{ end }} + scheme: {{ if or .Values.mtls.enabled ($tls.enabled | default false) }}HTTPS{{ else }}HTTP{{ end }} {{- end }} initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds | default 60 }} periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 5 }} @@ -376,7 +385,7 @@ readinessProbe: httpGet: path: /health/ready port: {{ .Values.service.endpoint.port | default 9000 }} - scheme: {{ if .Values.mtls.enabled }}HTTPS{{ else }}HTTP{{ end }} + scheme: {{ if or .Values.mtls.enabled ($tls.enabled | default false) }}HTTPS{{ else }}HTTP{{ end }} {{- end }} initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 60 }} periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 5 }} diff --git a/helm/rustfs/templates/configmap.yaml b/helm/rustfs/templates/configmap.yaml index 46e8331f8..e5aaa352f 100644 --- a/helm/rustfs/templates/configmap.yaml +++ b/helm/rustfs/templates/configmap.yaml @@ -1,3 +1,8 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} +{{- if and $tlsEnabled .Values.mtls.enabled -}} +{{- fail "tls.enabled and mtls.enabled are mutually exclusive; use ingress TLS for public access to an mTLS backend" -}} +{{- end -}} apiVersion: v1 kind: ConfigMap metadata: @@ -41,8 +46,12 @@ data: {{- end }} {{- if .Values.mtls.enabled }} RUSTFS_SERVER_MTLS_ENABLE: "1" - RUSTFS_TLS_KEYLOG: "1" + {{- end }} + {{- if or .Values.mtls.enabled $tlsEnabled }} + RUSTFS_TLS_KEYLOG: "0" RUSTFS_TLS_PATH: "/opt/tls" + RUSTFS_TLS_RELOAD_ENABLE: "1" + RUSTFS_TLS_RELOAD_INTERVAL: {{ $tls.reloadInterval | default 30 | quote }} {{- end }} {{- if .Values.config.rustfs.obs_endpoint.enabled }} RUSTFS_OBS_ENDPOINT: {{ .Values.config.rustfs.obs_endpoint.base_endpoint | quote }} diff --git a/helm/rustfs/templates/deployment.yaml b/helm/rustfs/templates/deployment.yaml index 39effeb24..97b663626 100644 --- a/helm/rustfs/templates/deployment.yaml +++ b/helm/rustfs/templates/deployment.yaml @@ -1,3 +1,5 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} @@ -123,25 +125,10 @@ spec: {{- toYaml .Values.resources | nindent 12 }} {{- include "rustfs.probes" . | nindent 10 }} volumeMounts: - {{- if .Values.mtls.enabled }} - - name: server-cert - mountPath: /opt/tls/rustfs_cert.pem - subPath: rustfs_cert.pem - - name: server-cert - mountPath: /opt/tls/rustfs_key.pem - subPath: rustfs_key.pem - - name: server-cert - mountPath: /opt/tls/ca.crt - subPath: ca.crt - - name: client-cert - mountPath: /opt/tls/client_cert.pem - subPath: client_cert.pem - - name: client-cert - mountPath: /opt/tls/client_key.pem - subPath: client_key.pem - - name: client-cert - mountPath: /opt/tls/client_ca.crt - subPath: client_ca.crt + {{- if or .Values.mtls.enabled $tlsEnabled }} + - name: tls-material + mountPath: /opt/tls + readOnly: true {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -157,27 +144,40 @@ spec: {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.mtls.enabled }} - - name: server-cert - secret: - secretName: {{ include "rustfs.fullname" . }}-server-tls - items: - - key: tls.crt - path: rustfs_cert.pem - - key: tls.key - path: rustfs_key.pem - - key: ca.crt - path: ca.crt - - name: client-cert - secret: - secretName: {{ include "rustfs.fullname" . }}-client-tls - items: - - key: tls.crt - path: client_cert.pem - - key: tls.key - path: client_key.pem - - key: ca.crt - path: client_ca.crt + {{- if or .Values.mtls.enabled $tlsEnabled }} + - name: tls-material + projected: + defaultMode: 0440 + sources: + - secret: + name: {{ if .Values.mtls.enabled }}{{ include "rustfs.fullname" . }}-server-tls{{ else }}{{ required "tls.existingSecret is required when tls.enabled=true" $tls.existingSecret }}{{ end }} + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem + {{- if .Values.mtls.enabled }} + - key: ca.crt + path: ca.crt + {{- end }} + {{- if .Values.mtls.enabled }} + - secret: + name: {{ include "rustfs.fullname" . }}-client-tls + items: + - key: tls.crt + path: client_cert.pem + - key: tls.key + path: client_key.pem + - key: ca.crt + path: client_ca.crt + {{- end }} + {{- if and $tlsEnabled $tls.trustSecret }} + - secret: + name: {{ $tls.trustSecret }} + items: + - key: ca.crt + path: public.crt + {{- end }} {{- end }} {{- if $logDirEnabled }} - name: logs diff --git a/helm/rustfs/templates/statefulset.yaml b/helm/rustfs/templates/statefulset.yaml index f2c90877d..54de97694 100644 --- a/helm/rustfs/templates/statefulset.yaml +++ b/helm/rustfs/templates/statefulset.yaml @@ -1,3 +1,5 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} {{- $poolsEnabled := and .Values.pools .Values.pools.enabled }} @@ -217,25 +219,10 @@ spec: {{- toYaml $.Values.resources | nindent 12 }} {{- include "rustfs.probes" $ | nindent 10 }} volumeMounts: - {{- if $.Values.mtls.enabled }} - - name: client-cert - mountPath: /opt/tls/client_cert.pem - subPath: client_cert.pem - - name: client-cert - mountPath: /opt/tls/client_key.pem - subPath: client_key.pem - - name: client-cert - mountPath: /opt/tls/client_ca.crt - subPath: client_ca.crt - - name: server-cert - mountPath: /opt/tls/rustfs_cert.pem - subPath: rustfs_cert.pem - - name: server-cert - mountPath: /opt/tls/rustfs_key.pem - subPath: rustfs_key.pem - - name: server-cert - mountPath: /opt/tls/ca.crt - subPath: ca.crt + {{- if or $.Values.mtls.enabled $tlsEnabled }} + - name: tls-material + mountPath: /opt/tls + readOnly: true {{- end }} {{- with $.Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -254,29 +241,42 @@ spec: - name: data mountPath: /data {{- end }} - {{- if or $.Values.mtls.enabled $.Values.extraVolumes }} + {{- if or $.Values.mtls.enabled $tlsEnabled $.Values.extraVolumes }} volumes: - {{- if $.Values.mtls.enabled }} - - name: server-cert - secret: - secretName: {{ include "rustfs.fullname" $ }}-server-tls - items: - - key: tls.crt - path: rustfs_cert.pem - - key: tls.key - path: rustfs_key.pem - - key: ca.crt - path: ca.crt - - name: client-cert - secret: - secretName: {{ include "rustfs.fullname" $ }}-client-tls - items: - - key: tls.crt - path: client_cert.pem - - key: tls.key - path: client_key.pem - - key: ca.crt - path: client_ca.crt + {{- if or $.Values.mtls.enabled $tlsEnabled }} + - name: tls-material + projected: + defaultMode: 0440 + sources: + - secret: + name: {{ if $.Values.mtls.enabled }}{{ include "rustfs.fullname" $ }}-server-tls{{ else }}{{ required "tls.existingSecret is required when tls.enabled=true" $tls.existingSecret }}{{ end }} + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem + {{- if $.Values.mtls.enabled }} + - key: ca.crt + path: ca.crt + {{- end }} + {{- if $.Values.mtls.enabled }} + - secret: + name: {{ include "rustfs.fullname" $ }}-client-tls + items: + - key: tls.crt + path: client_cert.pem + - key: tls.key + path: client_key.pem + - key: ca.crt + path: client_ca.crt + {{- end }} + {{- if and $tlsEnabled $tls.trustSecret }} + - secret: + name: {{ $tls.trustSecret }} + items: + - key: ca.crt + path: public.crt + {{- end }} {{- end }} {{- with $.Values.extraVolumes }} {{- toYaml . | nindent 8 }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 88d01e341..de0c4b7f1 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -388,6 +388,14 @@ gatewayApi: name: "" namespace: "" +# Native TLS identity managed by an external certificate controller. +# Public TLS and an mTLS backend use separate listeners or an ingress terminator. +tls: + enabled: false + existingSecret: "" # kubernetes.io/tls: tls.crt and tls.key only + trustSecret: "" # Optional independent outbound trust bundle: ca.crt + reloadInterval: 30 + mtls: enabled: false clientCertPath: "/opt/tls/client_cert.pem" diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 1f3f92469..fd9afef19 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -70,6 +70,7 @@ pub mod system; pub mod table_catalog; mod target_descriptor; pub mod tier; +pub mod tls_certificate; pub mod tls_debug; pub mod trace; pub mod usage_prefix; diff --git a/rustfs/src/admin/handlers/tls_certificate.rs b/rustfs/src/admin/handlers/tls_certificate.rs new file mode 100644 index 000000000..a45762625 --- /dev/null +++ b/rustfs/src/admin/handlers/tls_certificate.rs @@ -0,0 +1,177 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::{ + auth::authorize_admin_request, + router::{AdminOperation, Operation, S3Router}, + storage_api::s3::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}, +}; +use crate::server::{ + ADMIN_PREFIX, NativeTlsTransport, + tls_publish::{self, PublishError, PublishRequest}, +}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use rustfs_policy::policy::{action::Action, action::AdminAction}; + +pub fn register_tls_certificate_routes(router: &mut S3Router) -> std::io::Result<()> { + router.insert( + Method::GET, + &format!("{ADMIN_PREFIX}/v3/tls/certificate-state"), + AdminOperation(&CertificateStateHandler), + )?; + router.insert( + Method::POST, + &format!("{ADMIN_PREFIX}/v3/tls/certificate"), + AdminOperation(&PublishCertificateHandler), + )?; + Ok(()) +} + +pub struct CertificateStateHandler; +pub struct PublishCertificateHandler; + +fn require_native_tls(req: &S3Request) -> S3Result<()> { + if !req + .extensions + .get::() + .is_some_and(|transport| transport.0) + { + return Err(S3Error::with_message( + S3ErrorCode::AccessDenied, + "certificate management requires TLS on the RustFS listener", + )); + } + Ok(()) +} + +async fn authorize_certificate_request(req: &S3Request, action: AdminAction) -> S3Result<()> { + if req.credentials.is_none() { + return Err(S3Error::with_message( + S3ErrorCode::AccessDenied, + "certificate management requires credentials", + )); + } + authorize_admin_request(req, vec![Action::AdminAction(action)]).await?; + require_native_tls(req) +} + +fn tls_path() -> S3Result { + crate::server::tls_material::active_server_tls_path() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "certificate publication is not configured")) +} + +fn publication_error(error: PublishError) -> S3Error { + let (code, status) = match error { + PublishError::Conflict => (S3ErrorCode::InvalidRequest, StatusCode::CONFLICT), + PublishError::Disabled => (S3ErrorCode::InvalidRequest, StatusCode::SERVICE_UNAVAILABLE), + PublishError::Invalid => (S3ErrorCode::InvalidRequest, StatusCode::BAD_REQUEST), + PublishError::Storage | PublishError::Busy => (S3ErrorCode::InternalError, StatusCode::SERVICE_UNAVAILABLE), + }; + let mut result = S3Error::with_message(code, error.to_string()); + result.set_status_code(status); + result +} + +fn json_response(value: &impl serde::Serialize) -> S3Result> { + let bytes = serde_json::to_vec(value) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "certificate status serialization failed"))?; + let mut headers = HeaderMap::new(); + headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers)) +} + +#[async_trait::async_trait] +impl Operation for CertificateStateHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_certificate_request(&req, AdminAction::TLSCertificateStatusAdminAction).await?; + json_response(&tls_publish::certificate_state(tls_path()?).await.map_err(publication_error)?) + } +} + +#[async_trait::async_trait] +impl Operation for PublishCertificateHandler { + async fn call(&self, mut req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_certificate_request(&req, AdminAction::TLSCertificatePublishAdminAction).await?; + let body = req.input.store_all_limited(128 * 1024).await.map_err(|_| { + S3Error::with_message( + S3ErrorCode::InvalidRequest, + "certificate publication body exceeds the limit or is unreadable", + ) + })?; + let request: PublishRequest = serde_json::from_slice(&body) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "invalid certificate publication JSON"))?; + json_response(&tls_publish::publish(tls_path()?, request).await.map_err(publication_error)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(method: Method) -> S3Request { + S3Request { + input: Body::empty(), + method, + uri: "/rustfs/admin/v3/tls/certificate".parse().expect("URI"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } + } + + #[tokio::test] + async fn certificate_handlers_reject_missing_credentials() { + assert_eq!( + CertificateStateHandler + .call(request(Method::GET), Params::new()) + .await + .expect_err("unauthenticated GET") + .code(), + &S3ErrorCode::AccessDenied + ); + assert_eq!( + PublishCertificateHandler + .call(request(Method::POST), Params::new()) + .await + .expect_err("unauthenticated POST") + .code(), + &S3ErrorCode::AccessDenied + ); + } + + #[test] + fn forwarded_protocol_never_authorizes_private_key_upload() { + let mut request = request(Method::POST); + request.headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + assert!(require_native_tls(&request).is_err()); + request.extensions.insert(NativeTlsTransport(false)); + assert!(require_native_tls(&request).is_err()); + request.extensions.insert(NativeTlsTransport(true)); + assert!(require_native_tls(&request).is_ok()); + } + + #[test] + fn unknown_json_fields_are_rejected_without_echoing_material() { + let error = serde_json::from_str::( + r#"{"schema_version":"x","resource_id":"x","expected_target_revision":"x","issuance_id":"00000000-0000-0000-0000-000000000001","certificate_pem":"cert","private_key_pem":"secret","extra":true}"#, + ); + assert!(error.is_err()); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d330221d6..b9fb0373d 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -103,6 +103,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() inspect_archive::register_inspect_archive_route(r)?; gateway_key_inventory::register_gateway_key_inventory_route(r)?; tls_debug::register_tls_debug_route(r)?; + handlers::tls_certificate::register_tls_certificate_routes(r)?; kms::register_kms_route(r)?; oidc::register_oidc_route(r)?; idp_compat::register_idp_compat_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index d3baf28d2..8acd5f313 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -21,6 +21,8 @@ const ALL_ADMIN: AdminActionRef = AdminActionRef::new("AllAdminActions"); const ADD_USER_TO_GROUP: AdminActionRef = AdminActionRef::new("AddUserToGroupAdminAction"); const ATTACH_POLICY: AdminActionRef = AdminActionRef::new("AttachPolicyAdminAction"); const CONFIG_UPDATE: AdminActionRef = AdminActionRef::new("ConfigUpdateAdminAction"); +const TLS_CERTIFICATE_PUBLISH: AdminActionRef = AdminActionRef::new("TLSCertificatePublishAdminAction"); +const TLS_CERTIFICATE_STATUS: AdminActionRef = AdminActionRef::new("TLSCertificateStatusAdminAction"); const CONSOLE_LOG: AdminActionRef = AdminActionRef::new("ConsoleLogAdminAction"); const COMMIT_TABLE: AdminActionRef = AdminActionRef::new("CommitTableAction"); const CREATE_POLICY: AdminActionRef = AdminActionRef::new("CreatePolicyAdminAction"); @@ -501,6 +503,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ ), admin(HttpMethod::Get, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), admin(HttpMethod::Put, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/tls/certificate-state", + TLS_CERTIFICATE_STATUS, + RouteRiskLevel::Sensitive, + ), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/tls/certificate", + TLS_CERTIFICATE_PUBLISH, + RouteRiskLevel::High, + ), admin(HttpMethod::Get, "/rustfs/admin/v3/scanner/status", SERVER_INFO, RouteRiskLevel::Sensitive), admin( HttpMethod::Post, @@ -2233,6 +2247,14 @@ mod tests { } } + #[test] + fn route_policy_records_dedicated_tls_actions() { + assert_action(HttpMethod::Get, "/rustfs/admin/v3/tls/certificate-state", TLS_CERTIFICATE_STATUS); + assert_action(HttpMethod::Post, "/rustfs/admin/v3/tls/certificate", TLS_CERTIFICATE_PUBLISH); + assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/tls/certificate", CONFIG_UPDATE); + assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/tls/certificate-state", TLS_CERTIFICATE_PUBLISH); + } + /// `Critical` marks the routes whose worst case is permanent loss of user /// data, so the inventory is pinned in both directions: the KMS key /// deletion route may not be quietly downgraded, and no other route may be diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 3d85e3c11..ac24a3d6f 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -302,6 +302,8 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::PUT, "/v3/restore-config-history-kv"), admin_route(Method::GET, "/v3/config"), admin_route(Method::PUT, "/v3/config"), + admin_route(Method::GET, "/v3/tls/certificate-state"), + admin_route(Method::POST, "/v3/tls/certificate"), admin_route(Method::GET, "/v3/scanner/status"), admin_route(Method::POST, "/v3/scanner/cycle-state/reset"), admin_route(Method::POST, "/v3/scanner/usage-state/reset"), diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 8feed740a..c93f7bdd1 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -32,12 +32,12 @@ use crate::server::{ ssec_transport::SsecTransportLayer, strip_valid_port_suffix, tls_material::{ - TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, build_acceptor_from_loaded, + PreparedServerTls, TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, load_tls_material, spawn_reload_loop, }, }; #[cfg(feature = "http3")] -use crate::server::{http3, hybrid::HybridBody, tls_material::build_http3_server_config_from_loaded}; +use crate::server::{http3, hybrid::HybridBody}; use crate::storage_api::server::http as storage; use crate::storage_api::server::http::rpc::InternodeRpcService; #[cfg(test)] @@ -1448,6 +1448,7 @@ pub async fn start_http_server( // Load TLS materials and build server acceptor in a single pass. // Outbound material (root CAs, mTLS identity) was already published in main.rs; // this load is needed for the server-side TLS acceptor and reload loop. + let mut tls_initial_snapshot = None; let tls_acceptor = if tls_path_configured { let snapshot = load_tls_material(tls_path).await.map_err(|e| { Error::other(format!( @@ -1456,27 +1457,30 @@ pub async fn start_http_server( )) })?; - #[cfg(feature = "http3")] - if http3_enabled { - http3_server_config = build_http3_server_config_from_loaded(snapshot.server.as_ref(), std::path::Path::new(tls_path)) - .await - .map_err(|e| Error::other(format!("HTTP/3 TLS configuration failed: {e}")))?; - } - - let acceptor = build_acceptor_from_loaded(snapshot.server, std::path::Path::new(tls_path)) - .await - .map_err(|e| Error::other(e.to_string()))?; + let prepared = PreparedServerTls::from_snapshot(&snapshot).map_err(|e| Error::other(e.to_string()))?; // Fail closed: if TLS was explicitly configured but no server certificates // were found, refuse to start rather than silently falling back to plain HTTP. - match acceptor { + match prepared { None => { return Err(Error::other(format!( "TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but no server certificates were found", tls_path ))); } - Some(a) => Some(a), + Some(prepared) => { + #[cfg(feature = "http3")] + if http3_enabled { + http3_server_config = Some( + prepared + .http3_config() + .map_err(|e| Error::other(format!("HTTP/3 TLS configuration failed: {e}")))?, + ); + } + let acceptor = prepared.tcp_acceptor().map_err(|e| Error::other(e.to_string()))?; + tls_initial_snapshot = Some((snapshot, prepared)); + Some(acceptor) + } } } else { None @@ -1755,14 +1759,20 @@ pub async fn start_http_server( } else { None }; - let tls_reload_task = tls_acceptor.as_ref().and_then(|holder| { + let tls_reload_task = if let (Some(holder), Some((initial, prepared))) = (tls_acceptor.as_ref(), tls_initial_snapshot.take()) + { spawn_reload_loop( tls_path.to_string(), holder.clone(), + initial, + prepared, #[cfg(feature = "http3")] http3_server.as_ref().map(|(endpoint, _)| endpoint.clone()), ) - }); + .map_err(|e| Error::other(e.to_string()))? + } else { + None + }; let is_console = config.console_enable; let server_domains_configured = !config.server_domains.is_empty(); diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 7f919d6f7..4841bf4fe 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -29,6 +29,7 @@ pub(crate) mod runtime_sources; mod service_state; mod ssec_transport; pub mod tls_material; +pub(crate) mod tls_publish; #[cfg(feature = "http3")] mod http3; @@ -85,6 +86,7 @@ pub(crate) use readiness::collect_node_readiness_report; pub use readiness::publish_ready_when_runtime_ready; pub(crate) use readiness::snapshot_dependency_readiness_report; pub(crate) use readiness::{collect_cluster_read_health_report, collect_cluster_write_health_report}; +pub(crate) use ssec_transport::NativeTlsTransport; pub use crate::shared_types::RemoteAddr; diff --git a/rustfs/src/server/ssec_transport.rs b/rustfs/src/server/ssec_transport.rs index bcda8f08a..bbe7adaba 100644 --- a/rustfs/src/server/ssec_transport.rs +++ b/rustfs/src/server/ssec_transport.rs @@ -63,6 +63,10 @@ pub(crate) const METRIC_SSEC_PLAINTEXT_REQUESTS_TOTAL: &str = "rustfs_ssec_plain type BoxError = Box; type BoxBody = http_body_util::combinators::UnsyncBoxBody; +/// Set only by the listener stack; proxy protocol assertions do not establish this marker. +#[derive(Clone, Copy)] +pub(crate) struct NativeTlsTransport(pub(crate) bool); + /// Every header that carries, or announces, a customer-provided key: the /// object's own triple and the copy-source triple that CopyObject and /// UploadPartCopy use to read an SSE-C source. @@ -191,7 +195,8 @@ where self.inner.poll_ready(cx) } - fn call(&mut self, req: Request) -> Self::Future { + fn call(&mut self, mut req: Request) -> Self::Future { + req.extensions_mut().insert(NativeTlsTransport(self.connection_is_tls)); if !carries_ssec_headers(req.headers()) || is_secure_transport(self.connection_is_tls, req.extensions().get::()) { diff --git a/rustfs/src/server/tls_material.rs b/rustfs/src/server/tls_material.rs index 4d6239eb2..1790ee2a9 100644 --- a/rustfs/src/server/tls_material.rs +++ b/rustfs/src/server/tls_material.rs @@ -26,19 +26,20 @@ use rustfs_config::{ DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_RUSTFS_EXTRA_CA_CERT, ENV_SERVER_MTLS_ENABLE, ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, - ENV_TRUST_SYSTEM_CA, RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, - RUSTFS_TLS_CERT, -}; -use rustfs_tls_runtime::{ - ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions, - build_webpki_client_verifier, create_multi_cert_resolver, + ENV_TRUST_SYSTEM_CA, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, }; +#[cfg(test)] +use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME}; +use rustfs_tls_runtime::{ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, create_multi_cert_resolver}; +#[cfg(test)] +use rustfs_tls_runtime::{WebPkiClientVerifierOptions, build_webpki_client_verifier}; use rustfs_utils::{get_env_bool, get_env_opt_str}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Cursor; use std::path::{Path, PathBuf}; -use std::sync::Arc; use std::sync::RwLock; +use std::sync::{Arc, LazyLock, Mutex, Weak}; use std::time::Duration; use tokio_rustls::TlsAcceptor; use tracing::{debug, info, warn}; @@ -75,11 +76,20 @@ pub async fn load_tls_material(tls_path: &str) -> Result Result, tls_dir: &Path, @@ -100,76 +111,166 @@ pub(crate) async fn build_acceptor_from_loaded( ) .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - match server { - Some(RuntimeServerTlsMaterial::SingleCert { certs, key }) => { - let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier, ServerProtocol::Tcp)?; - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_acceptor_created", - mode = "single_cert", - "TLS acceptor created" - ); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) + build_acceptor_with_verifier(server, mtls_verifier) +} + +fn snapshot_client_verifier( + snapshot: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result>, TlsMaterialError> { + if !get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE) { + return Ok(None); + } + let pem = snapshot + .server_client_ca_pem + .as_deref() + .ok_or_else(|| TlsMaterialError::Parse("mTLS client trust is missing".into()))?; + let mut reader = Cursor::new(pem); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_reader_iter(&mut reader) { + roots + .add(cert.map_err(|e| TlsMaterialError::Parse(e.to_string()))?) + .map_err(|e| TlsMaterialError::Parse(e.to_string()))?; + } + let verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(roots)) + .build() + .map_err(|e| TlsMaterialError::Parse(e.to_string()))?; + #[cfg(test)] + TLS_PREPARE_COUNTS.with(|counts| { + let (roots, keys) = counts.get(); + counts.set((roots + 1, keys)); + }); + Ok(Some(verifier)) +} + +#[cfg(test)] +thread_local! { + static TLS_PREPARE_COUNTS: std::cell::Cell<(usize, usize)> = const { std::cell::Cell::new((0, 0)) }; +} + +/// Expensive certificate signing keys and client trust are prepared once per generation. +/// TCP and QUIC retain this shared resolver while owning their protocol settings. +#[derive(Clone)] +pub(crate) struct PreparedServerTls { + resolver: Arc, + verifier: Option>, + provider: Arc, +} + +impl PreparedServerTls { + pub(crate) fn from_snapshot(snapshot: &rustfs_tls_runtime::TlsMaterialSnapshot) -> Result, TlsMaterialError> { + Self::from_material(snapshot.server.as_ref(), snapshot_client_verifier(snapshot)?) + } + + fn from_material( + server: Option<&RuntimeServerTlsMaterial>, + verifier: Option>, + ) -> Result, TlsMaterialError> { + let Some(server) = server else { + return Ok(None); + }; + let provider = rustls::ServerConfig::builder().crypto_provider().clone(); + let (resolver, validity): (Arc, _) = match server { + RuntimeServerTlsMaterial::SingleCert { certs, key } => { + let validity = certificate_validities(std::iter::once(certs.as_slice()))?; + let certified = rustls::sign::CertifiedKey::from_der(certs.clone(), key.clone_key(), &provider) + .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?; + (Arc::new(rustls::sign::SingleCertAndKey::from(certified)), validity) + } + RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } => { + let validity = certificate_validities(cert_key_pairs.values().map(|(certs, _)| certs.as_slice()))?; + let pairs = cert_key_pairs + .iter() + .map(|(name, (certs, key))| (name.clone(), (certs.clone(), key.clone_key()))) + .collect(); + let resolver = create_multi_cert_resolver(pairs) + .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; + (Arc::new(resolver), validity) + } + }; + #[cfg(test)] + TLS_PREPARE_COUNTS.with(|counts| { + let (roots, keys) = counts.get(); + counts.set((roots, keys + 1)); + }); + Ok(Some(Self { + resolver: Arc::new(ValidityCheckedResolver { + inner: resolver, + validity, + }), + verifier, + provider, + })) + } + + pub(crate) fn tcp_acceptor(&self) -> Result, TlsMaterialError> { + let config = self.server_config(ServerProtocol::Tcp)?; + Ok(Arc::new(TlsAcceptorHolder::new(Arc::new(TlsAcceptor::from(Arc::new(config)))))) + } + + #[cfg(feature = "http3")] + pub(crate) fn http3_config(&self) -> Result { + let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(self.server_config(ServerProtocol::Http3)?) + .map_err(|e| TlsMaterialError::Io(format!("configure QUIC TLS: {e}")))?; + Ok(quinn::ServerConfig::with_crypto(Arc::new(crypto))) + } + + fn server_config(&self, protocol: ServerProtocol) -> Result { + let versions = match protocol { + ServerProtocol::Tcp => rustls::DEFAULT_VERSIONS, + #[cfg(feature = "http3")] + ServerProtocol::Http3 => &[&rustls::version::TLS13], + }; + let builder = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(versions) + .map_err(|e| TlsMaterialError::Io(format!("configure TLS protocol: {e}")))?; + let mut config = if let Some(verifier) = &self.verifier { + builder + .with_client_cert_verifier(verifier.clone()) + .with_cert_resolver(self.resolver.clone()) + } else { + builder.with_no_client_auth().with_cert_resolver(self.resolver.clone()) + }; + config.alpn_protocols = match protocol { + ServerProtocol::Tcp => vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()], + #[cfg(feature = "http3")] + ServerProtocol::Http3 => vec![b"h3".to_vec()], + }; + #[cfg(feature = "http3")] + if matches!(protocol, ServerProtocol::Http3) { + config.max_early_data_size = 0; } - Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { - let resolver = create_multi_cert_resolver(cert_key_pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; - let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier, ServerProtocol::Tcp)?; - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_acceptor_created", - mode = "sni_resolver", - "TLS acceptor created" - ); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) + config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); + if tls_key_log() { + config.key_log = Arc::new(rustls::KeyLogFile::new()); } - None => Ok(None), + Ok(config) } } -#[cfg(feature = "http3")] +#[cfg(test)] +fn build_acceptor_with_verifier( + server: Option, + mtls_verifier: Option>, +) -> Result>, TlsMaterialError> { + PreparedServerTls::from_material(server.as_ref(), mtls_verifier)? + .map(|prepared| prepared.tcp_acceptor()) + .transpose() +} + +#[cfg(all(test, feature = "http3"))] pub(crate) async fn build_http3_server_config_from_loaded( server: Option<&RuntimeServerTlsMaterial>, tls_dir: &Path, ) -> Result, TlsMaterialError> { - let Some(server) = server else { - return Ok(None); - }; - let mtls_verifier = build_webpki_client_verifier( WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) .build(), ) .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - - let source = match server { - RuntimeServerTlsMaterial::SingleCert { certs, key } => ServerCertSource::SingleCert { - certs: certs.clone(), - key: key.clone_key(), - }, - RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } => { - let pairs = cert_key_pairs - .iter() - .map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key()))) - .collect(); - - let resolver = create_multi_cert_resolver(pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build HTTP/3 SNI resolver: {e}")))?; - - ServerCertSource::Resolver(Arc::new(resolver)) - } - }; - - let tls = build_server_config(source, mtls_verifier, ServerProtocol::Http3)?; - let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(tls) - .map_err(|e| TlsMaterialError::Io(format!("configure QUIC TLS: {e}")))?; - - Ok(Some(quinn::ServerConfig::with_crypto(Arc::new(crypto)))) + PreparedServerTls::from_material(server, mtls_verifier)? + .map(|prepared| prepared.http3_config()) + .transpose() } // ── Outbound Enrichment ── @@ -179,18 +280,30 @@ pub(crate) async fn build_http3_server_config_from_loaded( /// 1. Optional: server leaf cert as root CA (`RUSTFS_TRUST_LEAF_CERT_AS_CA`) /// 2. Optional: system root CAs from platform paths (`RUSTFS_TRUST_SYSTEM_CA`) /// 3. Optional: mTLS identity from env-var-overridden paths -async fn enrich_outbound(outbound: &mut rustfs_tls_runtime::OutboundTlsMaterial, tls_dir: &Path) -> Result<(), TlsMaterialError> { - // 1. Optional: load leaf certs as root CAs - if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) - && load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut outbound.root_ca_pem).await - { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_root_ca_enriched", - source = "leaf_certificate", - "TLS root CA set enriched" - ); +async fn enrich_outbound(snapshot: &mut rustfs_tls_runtime::TlsMaterialSnapshot) -> Result<(), TlsMaterialError> { + let tls_dir = &snapshot.source.base_dir; + let outbound = &mut snapshot.outbound; + if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) { + let chains: Vec<_> = match &snapshot.server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, .. }) => vec![certs], + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { + cert_key_pairs.values().map(|(certs, _)| certs).collect() + } + None => Vec::new(), + }; + let mut certificates = BTreeSet::new(); + for chain in chains { + certificates.extend(chain.iter().map(|cert| cert.as_ref().to_vec())); + } + for certificate in certificates { + outbound.root_ca_pem.extend_from_slice( + format!( + "-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n", + base64_simd::STANDARD.encode_to_string(&certificate), + ) + .as_bytes(), + ); + } } // 2. Optional: load system root CAs @@ -260,19 +373,6 @@ async fn load_mtls_identity_from_overridden_paths( None => tls_dir.join(RUSTFS_CLIENT_KEY_FILENAME), }; - if !client_cert_path.exists() || !client_key_path.exists() { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "mtls_identity_state", - state = "not_configured", - cert_path = ?client_cert_path, - key_path = ?client_key_path, - "mTLS client identity not configured" - ); - return Ok(None); - } - let cert_pem = tokio::fs::read(&client_cert_path) .await .map_err(|e| TlsMaterialError::Io(format!("read client cert {client_cert_path:?}: {e}")))?; @@ -281,11 +381,13 @@ async fn load_mtls_identity_from_overridden_paths( .map_err(|e| TlsMaterialError::Io(format!("read client key {client_key_path:?}: {e}")))?; let mut reader = std::io::Cursor::new(&cert_pem); - if CertificateDer::pem_reader_iter(&mut reader).next().is_none() { - return Err(TlsMaterialError::Parse("no valid certificate in client cert PEM".into())); - } + let certs = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| TlsMaterialError::Parse(format!("invalid client certificate: {e}")))?; let mut reader = std::io::Cursor::new(&key_pem); - PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| TlsMaterialError::Parse(format!("invalid client key PEM: {e}")))?; + let key = + PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| TlsMaterialError::Parse(format!("invalid client key: {e}")))?; + rustfs_tls_runtime::validate_certified_key(&certs, &key, "client identity override", false).map_err(map_runtime_tls_error)?; info!( component = LOG_COMPONENT_TLS, @@ -413,54 +515,47 @@ async fn load_cert_file(path: &Path, pem_data: &mut Vec, desc: &str) -> bool /// Search for and load certificate files matching `cert_name` in the directory /// and one level of subdirectories. /// Returns `true` if at least one matching file was loaded. -async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec) -> bool { - let Ok(mut rd) = tokio::fs::read_dir(dir).await else { - debug!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_material_directory_state", - path = %dir.display(), - state = "missing", - "TLS material directory state changed" + +type CertificateValidities = HashMap, (i64, i64)>; + +fn certificate_validities<'a>( + chains: impl Iterator]>, +) -> Result { + let mut validity = HashMap::new(); + for chain in chains { + let leaf = chain + .first() + .ok_or_else(|| TlsMaterialError::Parse("empty certificate chain".into()))?; + let (_, parsed) = x509_parser::parse_x509_certificate(leaf.as_ref()) + .map_err(|e| TlsMaterialError::Parse(format!("invalid certificate: {e}")))?; + validity.insert( + leaf.as_ref().to_vec(), + (parsed.validity().not_before.timestamp(), parsed.validity().not_after.timestamp()), ); - return false; - }; - - let mut loaded = false; - while let Ok(Some(entry)) = rd.next_entry().await { - let Ok(ft) = entry.file_type().await else { continue }; - - if ft.is_file() { - let fname = entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&entry.path(), pem_data, "certificate").await { - loaded = true; - } - } else if ft.is_dir() - && let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await - { - while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { - if let Ok(sub_ft) = sub_entry.file_type().await - && sub_ft.is_file() - { - let fname = sub_entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await { - loaded = true; - } - } - } - } } - loaded + Ok(validity) } -// ── Server Config Construction ── +#[derive(Debug)] +struct ValidityCheckedResolver { + inner: Arc, + validity: CertificateValidities, +} -enum ServerCertSource { - Resolver(Arc), - SingleCert { - certs: Vec>, - key: PrivateKeyDer<'static>, - }, +impl rustls::server::ResolvesServerCert for ValidityCheckedResolver { + fn resolve(&self, hello: rustls::server::ClientHello<'_>) -> Option> { + let certificate = self.inner.resolve(hello)?; + let leaf = certificate.cert.first()?; + let (not_before, not_after) = self.validity.get(leaf.as_ref())?; + let now = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(), + ) + .ok()?; + (*not_before <= now && now < *not_after).then_some(certificate) + } } enum ServerProtocol { @@ -469,62 +564,6 @@ enum ServerProtocol { Http3, } -fn build_server_config( - cert_source: ServerCertSource, - mtls_verifier: Option>, - protocol: ServerProtocol, -) -> Result { - let builder = match protocol { - ServerProtocol::Tcp => rustls::ServerConfig::builder(), - #[cfg(feature = "http3")] - ServerProtocol::Http3 => rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]), - }; - - let mut config = match cert_source { - ServerCertSource::Resolver(resolver) => { - if let Some(verifier) = mtls_verifier { - builder.with_client_cert_verifier(verifier).with_cert_resolver(resolver) - } else { - builder.with_no_client_auth().with_cert_resolver(resolver) - } - } - ServerCertSource::SingleCert { certs, key } => { - if let Some(verifier) = mtls_verifier { - builder - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))? - } else { - builder - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))? - } - } - }; - - config.alpn_protocols = match protocol { - ServerProtocol::Tcp => { - vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()] - } - #[cfg(feature = "http3")] - ServerProtocol::Http3 => vec![b"h3".to_vec()], - }; - - #[cfg(feature = "http3")] - if matches!(protocol, ServerProtocol::Http3) { - config.max_early_data_size = 0; - } - - config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); - - if tls_key_log() { - config.key_log = Arc::new(rustls::KeyLogFile::new()); - } - - Ok(config) -} - fn tls_key_log() -> bool { get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG) } @@ -647,26 +686,30 @@ impl TlsAcceptorHolder { } // Build both configurations before publishing either listener's new certificate. +#[cfg(all(test, feature = "http3"))] async fn reload_acceptors( server: Option, tls_dir: &Path, holder: &TlsAcceptorHolder, #[cfg(feature = "http3")] http3_endpoint: Option<&quinn::Endpoint>, ) -> Result { - #[cfg(feature = "http3")] + let verifier = build_webpki_client_verifier( + WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) + .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) + .build(), + ) + .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; + let Some(prepared) = PreparedServerTls::from_material(server.as_ref(), verifier)? else { + return Ok(false); + }; let http3_config = if http3_endpoint.is_some() { - let config = build_http3_server_config_from_loaded(server.as_ref(), tls_dir).await?; - let Some(mut config) = config else { - return Ok(false); - }; + let mut config = prepared.http3_config()?; config.migration(false); Some(config) } else { None }; - let Some(new_holder) = build_acceptor_from_loaded(server, tls_dir).await? else { - return Ok(false); - }; + let new_holder = prepared.tcp_acceptor()?; #[cfg(feature = "http3")] if let (Some(endpoint), Some(config)) = (http3_endpoint, http3_config) { endpoint.set_server_config(Some(config)); @@ -680,146 +723,327 @@ async fn reload_acceptors( /// Spawn a background task that periodically checks for TLS certificate changes. /// Single load per tick: loads once via tls-runtime, enriches, publishes outbound, /// and builds acceptor — no double reads. +#[derive(Clone)] +struct ReloadTarget { + holder: Arc, + #[cfg(feature = "http3")] + http3_endpoint: Option, +} + +struct ReloadState { + targets: HashMap, + accepted: Arc, + tcp: Arc, + prepared: PreparedServerTls, + #[cfg(feature = "http3")] + quic: Option, +} + +struct ReloadGroup { + required_startup_files: BTreeSet, + state: Mutex, + changed: tokio::sync::Notify, + next_id: std::sync::atomic::AtomicU64, + #[cfg(test)] + pass_completed: tokio::sync::Notify, +} + +// Lock order is registry, then group state, then acceptor. Neither guard crosses an await. +static RELOAD_GROUPS: LazyLock>>> = LazyLock::new(|| Mutex::new(HashMap::new())); + +pub(crate) fn observed_server_snapshot(tls_path: &str) -> Option> { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + let group = registry.get(tls_path)?.upgrade()?; + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.targets.is_empty()).then(|| state.accepted.clone()) +} + +/// Resolve the source actually registered by listener configuration, including CLI overrides. +pub(crate) fn active_server_tls_path() -> Option { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + let mut paths = registry.iter().filter_map(|(path, group)| { + let group = group.upgrade()?; + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.targets.is_empty()).then(|| path.clone()) + }); + let path = paths.next()?; + paths.next().is_none().then_some(path) +} + +pub(crate) fn notify_server_publication(tls_path: &str) { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(group) = registry.get(tls_path).and_then(Weak::upgrade) { + group.changed.notify_one(); + } +} + +struct ReloadRegistration { + group: Arc, + id: u64, +} + +impl Drop for ReloadRegistration { + fn drop(&mut self) { + self.group + .state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .targets + .remove(&self.id); + self.group.changed.notify_one(); + } +} + +fn server_names(server: &Option) -> Result>, TlsMaterialError> { + let chains: Vec<_> = match server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, .. }) => vec![("default", certs)], + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => cert_key_pairs + .iter() + .map(|(selector, (certs, _))| (selector.as_str(), certs)) + .collect(), + None => Vec::new(), + }; + let mut selectors = BTreeMap::new(); + for (selector, chain) in chains { + let leaf = chain + .first() + .ok_or_else(|| TlsMaterialError::Parse("empty certificate chain".into()))?; + selectors.insert(selector.to_string(), certificate_names(leaf)?); + } + Ok(selectors) +} + +pub(crate) fn certificate_names(leaf: &CertificateDer<'_>) -> Result, TlsMaterialError> { + let mut names = BTreeSet::new(); + let (_, parsed) = x509_parser::parse_x509_certificate(leaf.as_ref()) + .map_err(|e| TlsMaterialError::Parse(format!("invalid certificate: {e}")))?; + if let Some(san) = parsed + .subject_alternative_name() + .map_err(|e| TlsMaterialError::Parse(format!("invalid SAN: {e}")))? + { + for name in &san.value.general_names { + match name { + x509_parser::extensions::GeneralName::DNSName(name) => { + names.insert(format!("dns:{}", name.to_ascii_lowercase())); + } + x509_parser::extensions::GeneralName::IPAddress(bytes) => { + names.insert(format!("ip:{bytes:?}")); + } + _ => {} + } + } + } + Ok(names) +} + +pub(crate) fn validate_required_material_files( + required: &BTreeSet, + candidate: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result<(), TlsMaterialError> { + if !required.is_subset(&candidate.material_files) { + return Err(TlsMaterialError::Parse("required TLS source components disappeared".into())); + } + Ok(()) +} + +pub(crate) fn validate_candidate_replacement( + accepted: &rustfs_tls_runtime::TlsMaterialSnapshot, + candidate: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result<(), TlsMaterialError> { + validate_required_material_files(&accepted.material_files, candidate)?; + let old_names = server_names(&accepted.server)?; + let new_names = server_names(&candidate.server)?; + for (selector, names) in old_names { + if !new_names.get(&selector).is_some_and(|new| names.is_subset(new)) { + return Err(TlsMaterialError::Parse("required server certificate names disappeared".into())); + } + } + Ok(()) +} + +fn outbound_material_matches(a: &rustfs_tls_runtime::OutboundTlsMaterial, b: &rustfs_tls_runtime::OutboundTlsMaterial) -> bool { + a.root_ca_pem == b.root_ca_pem && a.mtls_identity == b.mtls_identity +} + pub(crate) fn spawn_reload_loop( tls_path: String, holder: Arc, + initial: rustfs_tls_runtime::TlsMaterialSnapshot, + prepared: PreparedServerTls, #[cfg(feature = "http3")] http3_endpoint: Option, -) -> Option> { - let enabled = get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE); - if !enabled { - debug!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "disabled", - env_var = ENV_TLS_RELOAD_ENABLE, - "TLS reload state changed" - ); - return None; +) -> Result>, TlsMaterialError> { + if !get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE) { + return Ok(None); } + let required_startup_files = crate::startup_tls_material::required_startup_material_files(&tls_path); + let initial = Arc::new(initial); + let mut registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + registry.retain(|_, group| group.strong_count() != 0); + let existing = registry.get(&tls_path).and_then(Weak::upgrade); + let first = existing.is_none(); + let group = existing.unwrap_or_else(|| { + Arc::new(ReloadGroup { + required_startup_files, + state: Mutex::new(ReloadState { + targets: HashMap::new(), + accepted: initial.clone(), + tcp: holder.get(), + prepared, + #[cfg(feature = "http3")] + quic: None, + }), + changed: tokio::sync::Notify::new(), + next_id: std::sync::atomic::AtomicU64::new(1), + #[cfg(test)] + pass_completed: tokio::sync::Notify::new(), + }) + }); + let id = group.next_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + { + let mut state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + #[cfg(feature = "http3")] + if let Some(endpoint) = &http3_endpoint { + if state.quic.is_none() { + let mut config = state.prepared.http3_config()?; + config.migration(false); + state.quic = Some(config); + } + endpoint.set_server_config(state.quic.clone()); + } + holder.swap(&TlsAcceptorHolder::new(state.tcp.clone())); + state.targets.insert( + id, + ReloadTarget { + holder, + #[cfg(feature = "http3")] + http3_endpoint, + }, + ); + } + registry.insert(tls_path.clone(), Arc::downgrade(&group)); + drop(registry); + if first { + tokio::spawn(run_reload_group(tls_path, group.clone())); + } + group.changed.notify_one(); + let registration = ReloadRegistration { group, id }; + Ok(Some(tokio::spawn(async move { + let _registration = registration; + std::future::pending::<()>().await; + }))) +} +async fn run_reload_group(tls_path: String, group: Arc) { let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5); - let tls_source = TlsSource::from_directory(&tls_path); - - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "enabled", - interval_secs, - "TLS reload state changed" - ); - - Some(tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); - let tls_dir = PathBuf::from(&tls_path); - loop { - interval.tick().await; - - match rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source).await { - Ok(mut snapshot) => { - if let Err(e) = enrich_outbound(&mut snapshot.outbound, &tls_dir).await { - startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "enrich_err", None, None); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "enrich_outbound", - error = %e, - "TLS reload failed" - ); - continue; - } - - let generation = startup_runtime_sources::current_outbound_tls_generation().saturating_add(1); - startup_runtime_sources::publish_outbound_tls_state(TlsGeneration(generation), &snapshot.outbound).await; - startup_runtime_sources::record_tls_generation("rustfs_server_reload_loop", generation); - if !snapshot.outbound.root_ca_pem.is_empty() { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_root_ca_enriched", - source = "reload_snapshot", - generation, - "TLS root CA set updated" - ); - } - - match reload_acceptors( - snapshot.server, - &tls_dir, - &holder, - #[cfg(feature = "http3")] - http3_endpoint.as_ref(), - ) - .await - { - Ok(true) => { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "reloaded", - generation, - "TLS reload state changed" - ); - startup_runtime_sources::record_tls_reload_result( - "rustfs_server_reload_loop", - "ok", - None, - Some(generation), - ); - } - Ok(false) => { - startup_runtime_sources::record_tls_reload_skipped("rustfs_server_reload_loop", "no_acceptor"); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "build_acceptor", - reason = "no_acceptor", - "TLS reload returned no acceptor" - ) - } - Err(e) => { - startup_runtime_sources::record_tls_reload_result( - "rustfs_server_reload_loop", - "acceptor_err", - None, - Some(generation), - ); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "build_acceptor", - error = %e, - generation, - "TLS reload failed" - ) - } + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut generation = 1_u64; + loop { + tokio::select! { + _ = interval.tick() => {}, + _ = group.changed.notified() => {}, + } + let (accepted, empty) = { + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (state.accepted.clone(), state.targets.is_empty()) + }; + if empty { + let mut registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + if group.state.lock().unwrap_or_else(|e| e.into_inner()).targets.is_empty() { + registry.remove(&tls_path); + return; + } + continue; + } + let result = async { + let candidate = load_tls_material(&tls_path).await?; + validate_candidate_replacement(&accepted, &candidate)?; + validate_required_material_files(&group.required_startup_files, &candidate)?; + // Listener initialization can load a newer bundle than startup published. + let published = startup_runtime_sources::current_outbound_tls_state().await; + let required_roots = + !accepted.outbound.root_ca_pem.is_empty() || published.root_ca_pem.as_ref().is_some_and(|pem| !pem.is_empty()); + let required_identity = accepted.outbound.mtls_identity.is_some() || published.mtls_identity.is_some(); + if (required_roots && candidate.outbound.root_ca_pem.is_empty()) + || (required_identity && candidate.outbound.mtls_identity.is_none()) + { + return Err(TlsMaterialError::Parse("required outbound TLS material disappeared".into())); + } + let outbound_changed = !outbound_material_matches(&accepted.outbound, &candidate.outbound) + || published.root_ca_pem.as_deref().unwrap_or_default() != candidate.outbound.root_ca_pem.as_slice() + || published.mtls_identity != candidate.outbound.mtls_identity; + if accepted.fingerprint == candidate.fingerprint && !outbound_changed { + startup_runtime_sources::record_tls_reload_skipped("rustfs_server_reload_loop", "unchanged"); + return Ok(None); + } + let prepared = PreparedServerTls::from_snapshot(&candidate)? + .ok_or_else(|| TlsMaterialError::Parse("server certificate is missing".into()))?; + let tcp = prepared.tcp_acceptor()?; + #[cfg(feature = "http3")] + let quic = { + let mut config = prepared.http3_config()?; + config.migration(false); + config + }; + let candidate = Arc::new(candidate); + let listeners = { + let mut state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + // Newly registered listeners join the same prepared generation under this lock. + for target in state.targets.values() { + #[cfg(feature = "http3")] + if let Some(endpoint) = &target.http3_endpoint { + endpoint.set_server_config(Some(quic.clone())); } + target.holder.swap(&tcp); } - Err(e) => { - startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "load_err", None, None); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "load_snapshot", - error = %e, - "TLS reload failed" - ); + state.tcp = tcp.get(); + state.prepared = prepared; + state.accepted = candidate.clone(); + #[cfg(feature = "http3")] + { + state.quic = Some(quic); } + state.targets.len() + }; + if outbound_changed { + let next = startup_runtime_sources::current_outbound_tls_generation().saturating_add(1); + startup_runtime_sources::publish_outbound_tls_state(TlsGeneration(next), &candidate.outbound).await; + } + Ok(Some(listeners)) + } + .await; + match result { + Ok(Some(listeners)) => { + generation = generation.saturating_add(1); + startup_runtime_sources::record_tls_generation("rustfs_server_reload_loop", generation); + startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "ok", None, Some(generation)); + info!( + event = "tls_reload_state", + component = LOG_COMPONENT_TLS, + subsystem = LOG_SUBSYSTEM_TLS, + state = "reloaded", + generation, + listeners, + "TLS material updated" + ); + } + Ok(None) => {} + Err(error) => { + startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "rejected", None, None); + warn!(event = "tls_reload_failed", component = LOG_COMPONENT_TLS, subsystem = LOG_SUBSYSTEM_TLS, + result = "rejected", error = %error, "TLS material update rejected"); } } - })) + #[cfg(test)] + group.pass_completed.notify_one(); + } } #[cfg(test)] mod tests { use super::*; use rcgen::CertifiedKey; + use rustfs_config::RUSTFS_TLS_CERT; use std::fs; use std::sync::Once; use tempfile::TempDir; @@ -839,6 +1063,507 @@ mod tests { fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap(); } + #[tokio::test] + #[serial_test::serial] + async fn prepared_generation_shares_crypto_and_serves_sni_with_mtls() { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars([(ENV_SERVER_MTLS_ENABLE, Some("1"))], async { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "default.local"); + let sni_dir = dir.path().join("sni.example"); + fs::create_dir(&sni_dir).expect("SNI directory"); + write_test_cert_pair(&sni_dir, "sni.example"); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).expect("CA parameters"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca = ca_params.self_signed(&ca_key).expect("CA certificate"); + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca.pem()).expect("client trust"); + let mut client_params = rcgen::CertificateParams::new(vec!["client.example".into()]).expect("client parameters"); + client_params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; + let client_key = rcgen::KeyPair::generate().expect("client key"); + let issuer = rcgen::Issuer::from_params(&ca_params, &ca_key); + let client_cert = client_params.signed_by(&client_key, &issuer).expect("client identity"); + let snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .expect("validated TLS snapshot"); + let before = TLS_PREPARE_COUNTS.with(std::cell::Cell::get); + let prepared = PreparedServerTls::from_snapshot(&snapshot) + .expect("prepare generation") + .expect("server TLS"); + let tcp = prepared.server_config(ServerProtocol::Tcp).expect("TCP config"); + let another_tcp = prepared + .server_config(ServerProtocol::Tcp) + .expect("additional listener config"); + assert!(Arc::ptr_eq(&tcp.cert_resolver, &another_tcp.cert_resolver)); + #[cfg(feature = "http3")] + { + let quic_tls = prepared.server_config(ServerProtocol::Http3).expect("QUIC TLS config"); + assert!(Arc::ptr_eq(&tcp.cert_resolver, &quic_tls.cert_resolver)); + } + let after = TLS_PREPARE_COUNTS.with(std::cell::Cell::get); + assert_eq!( + (after.0 - before.0, after.1 - before.1), + (1, 1), + "client verifier and signing resolver must each be prepared once for all protocols and listeners" + ); + + let RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } = snapshot.server.as_ref().expect("SNI material") else { + panic!("multiple selectors expected"); + }; + let mut roots = rustls::RootCertStore::empty(); + for (certificates, _) in cert_key_pairs.values() { + roots.add(certificates[0].clone()).expect("server trust"); + } + for (name, selector) in [("default.local", "default"), ("sni.example", "sni.example")] { + let expected = cert_key_pairs[selector].0[0].clone(); + let client_tls = rustls::ClientConfig::builder() + .with_root_certificates(roots.clone()) + .with_client_auth_cert( + vec![client_cert.der().clone()], + PrivateKeyDer::try_from(client_key.serialize_der()).expect("client private key"), + ) + .expect("mTLS client"); + let (server_io, client_io) = tokio::io::duplex(65536); + let acceptor = prepared.tcp_acceptor().expect("acceptor").get(); + let connector = tokio_rustls::TlsConnector::from(Arc::new(client_tls.clone())); + let (server, client) = tokio::join!( + acceptor.accept(server_io), + connector.connect(rustls::pki_types::ServerName::try_from(name.to_owned()).expect("server name"), client_io) + ); + let server = server.expect("authenticated server handshake"); + let client = client.expect("authenticated client handshake"); + assert_eq!(client.get_ref().1.peer_certificates().expect("server identity")[0], expected); + assert_eq!(server.get_ref().1.peer_certificates().expect("client identity")[0], *client_cert.der()); + + #[cfg(feature = "http3")] + { + let endpoint = quinn::Endpoint::server( + prepared.http3_config().expect("QUIC config"), + "127.0.0.1:0".parse().expect("address"), + ) + .expect("QUIC server"); + let mut client_tls = client_tls; + client_tls.alpn_protocols = vec![b"h3".to_vec()]; + let crypto = quinn::crypto::rustls::QuicClientConfig::try_from(client_tls).expect("QUIC client TLS"); + let mut client_endpoint = + quinn::Endpoint::client("127.0.0.1:0".parse().expect("address")).expect("QUIC client"); + client_endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(crypto))); + let connection = client_endpoint + .connect(endpoint.local_addr().expect("server address"), name) + .expect("connect"); + let (server, client) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(async { endpoint.accept().await.expect("incoming").await }, connection) + }) + .await + .expect("QUIC handshake deadline"); + let server = server.expect("QUIC mTLS server handshake"); + let client = client.expect("QUIC mTLS client handshake"); + let identity = client + .peer_identity() + .expect("QUIC server identity") + .downcast::>>() + .expect("certificate identity"); + assert_eq!(identity[0], expected); + let identity = server + .peer_identity() + .expect("QUIC client identity") + .downcast::>>() + .expect("certificate identity"); + assert_eq!(identity[0], *client_cert.der()); + client.close(0u32.into(), b"done"); + server.close(0u32.into(), b"done"); + endpoint.close(0u32.into(), b"done"); + client_endpoint.close(0u32.into(), b"done"); + } + } + }) + .await; + } + + #[tokio::test] + async fn candidate_replacement_preserves_ca_components_and_default_names() { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "s3.example.com"); + let ca_a = rcgen::generate_simple_self_signed(vec!["ca-a.example".into()]).expect("CA A"); + let ca_b = rcgen::generate_simple_self_signed(vec!["ca-b.example".into()]).expect("CA B"); + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca_a.cert.pem()).expect("primary client CA"); + fs::write(dir.path().join(RUSTFS_CA_CERT), ca_b.cert.pem()).expect("fallback CA"); + let source = TlsSource::from_directory(dir.path()); + let accepted = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("accepted generation"); + fs::remove_file(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME)).expect("remove primary client CA"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("fallback generation parses"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "a fallback must not replace accepted client trust" + ); + + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca_a.cert.pem()).expect("restore primary CA"); + write_test_cert_pair(dir.path(), "wrong.example.com"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("valid but wrong SAN"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "default SAN coverage must survive renewal" + ); + + write_test_cert_pair(dir.path(), "s3.example.com"); + let renewed = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("valid renewal"); + validate_candidate_replacement(&accepted, &renewed).expect("renewal preserving names and trust sources"); + } + + #[tokio::test] + async fn candidate_default_names_cannot_move_into_another_sni_certificate() { + let dir = TempDir::new().expect("TLS directory"); + let sni = dir.path().join("example.com"); + fs::create_dir(&sni).expect("SNI directory"); + write_test_cert_pair(dir.path(), "default.local"); + write_test_cert_pair(&sni, "example.com"); + let source = TlsSource::from_directory(dir.path()); + let accepted = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("accepted selectors"); + write_test_cert_pair(dir.path(), "wrong.local"); + let relocated = rcgen::generate_simple_self_signed(vec!["example.com".into(), "default.local".into()]) + .expect("certificate with relocated SAN"); + fs::write(sni.join(RUSTFS_TLS_CERT), relocated.cert.pem()).expect("SNI certificate"); + fs::write(sni.join(rustfs_config::RUSTFS_TLS_KEY), relocated.signing_key.serialize_pem()).expect("SNI key"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("individually valid candidate"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "another selector's SAN does not cover the default resolver's hostname" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn snapshot_leaf_trust_is_encoded_in_canonical_der_order() { + let dir = TempDir::new().expect("TLS directory"); + let a = rcgen::generate_simple_self_signed(vec!["a.example".into()]).expect("certificate A"); + let b = rcgen::generate_simple_self_signed(vec!["b.example".into()]).expect("certificate B"); + let (first, second) = if a.cert.der().as_ref() > b.cert.der().as_ref() { + (&a, &b) + } else { + (&b, &a) + }; + fs::write(dir.path().join(RUSTFS_TLS_CERT), format!("{}{}", first.cert.pem(), second.cert.pem())) + .expect("descending DER chain"); + fs::write(dir.path().join(rustfs_config::RUSTFS_TLS_KEY), first.signing_key.serialize_pem()).expect("leaf key"); + let mut snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .expect("validated material"); + temp_env::async_with_vars( + [ + (ENV_TRUST_LEAF_CERT_AS_CA, Some("1")), + (ENV_TRUST_SYSTEM_CA, Some("0")), + (ENV_MTLS_CLIENT_CERT, None), + (ENV_MTLS_CLIENT_KEY, None), + ], + async { + enrich_outbound(&mut snapshot).await.expect("derive leaf trust"); + let mut reader = Cursor::new(&snapshot.outbound.root_ca_pem); + let certificates = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .expect("derived certificate PEM"); + assert_eq!( + certificates, + vec![second.cert.der().clone(), first.cert.der().clone()], + "leaf trust must not depend on source enumeration order" + ); + }, + ) + .await; + } + + #[tokio::test] + async fn client_identity_override_rejects_mismatch_and_missing_files() { + let dir = TempDir::new().expect("identity directory"); + let a = rcgen::generate_simple_self_signed(vec!["client-a.example".into()]).expect("identity A"); + let b = rcgen::generate_simple_self_signed(vec!["client-b.example".into()]).expect("identity B"); + let cert = dir.path().join(RUSTFS_CLIENT_CERT_FILENAME); + let key = dir.path().join(RUSTFS_CLIENT_KEY_FILENAME); + fs::write(&cert, a.cert.pem()).expect("client cert"); + fs::write(&key, b.signing_key.serialize_pem()).expect("mismatched key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .is_err() + ); + fs::remove_file(&key).expect("remove override key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .is_err() + ); + fs::write(&key, a.signing_key.serialize_pem()).expect("matching key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .expect("valid override") + .is_some() + ); + } + + enum StartupRotation { + Outbound, + RootsOnly, + IdentityOnly, + ServerOnly, + MissingRoots, + PartialRoots, + MissingIdentity, + } + + async fn assert_initial_outbound_reconciliation(rotation: StartupRotation) { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (ENV_TRUST_SYSTEM_CA, Some("0")), + (ENV_MTLS_CLIENT_CERT, None), + (ENV_MTLS_CLIENT_KEY, None), + (ENV_RUSTFS_EXTRA_CA_CERT, None), + (super::super::tls_publish::ENV_TLS_PUBLISH_DIR, None), + ], + async { + let previous = startup_runtime_sources::current_outbound_tls_state().await; + let previous_files = crate::startup_tls_material::replace_startup_material_files(None); + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "server.example"); + let identity_a = rcgen::generate_simple_self_signed(vec!["client-a.example".into()]).expect("identity A"); + fs::write(dir.path().join(RUSTFS_CA_CERT), identity_a.cert.pem()).expect("trust A"); + fs::write(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME), identity_a.cert.pem()).expect("client certificate A"); + fs::write(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME), identity_a.signing_key.serialize_pem()) + .expect("client key A"); + if matches!(rotation, StartupRotation::PartialRoots) { + let public_ca = + rcgen::generate_simple_self_signed(vec!["public-ca.example".into()]).expect("distinct public CA"); + fs::write(dir.path().join(rustfs_config::RUSTFS_PUBLIC_CERT), public_ca.cert.pem()) + .expect("additional public trust"); + } + let path = dir.path().to_str().expect("UTF8 path").to_string(); + let mut config = crate::config::Config::new("127.0.0.1:0", Vec::new()); + config.tls_path = Some(path.clone()); + crate::startup_tls_material::init_outbound_tls_material(&config) + .await + .expect("real startup publication"); + let startup = startup_runtime_sources::current_outbound_tls_state().await; + + write_test_cert_pair(dir.path(), "server.example"); + match rotation { + StartupRotation::Outbound | StartupRotation::RootsOnly | StartupRotation::IdentityOnly => { + let identity_b = rcgen::generate_simple_self_signed(vec!["client-b.example".into()]).expect("identity B"); + if !matches!(rotation, StartupRotation::IdentityOnly) { + fs::write(dir.path().join(RUSTFS_CA_CERT), identity_b.cert.pem()).expect("trust B"); + } + if !matches!(rotation, StartupRotation::RootsOnly) { + fs::write(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME), identity_b.cert.pem()) + .expect("client certificate B"); + fs::write(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME), identity_b.signing_key.serialize_pem()) + .expect("client key B"); + } + } + StartupRotation::ServerOnly => {} + StartupRotation::MissingRoots | StartupRotation::PartialRoots => { + fs::remove_file(dir.path().join(RUSTFS_CA_CERT)).expect("remove startup roots"); + } + StartupRotation::MissingIdentity => { + fs::remove_file(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME)).expect("remove startup client certificate"); + fs::remove_file(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME)).expect("remove startup client key"); + } + } + let initial = load_tls_material(&path).await.expect("HTTP initialization snapshot"); + let incomplete = matches!( + rotation, + StartupRotation::MissingRoots | StartupRotation::PartialRoots | StartupRotation::MissingIdentity + ); + let expected_roots = if incomplete { + startup.root_ca_pem.clone().expect("startup trust") + } else { + initial.outbound.root_ca_pem.clone() + }; + let expected_identity = if incomplete { + startup.mtls_identity.clone() + } else { + initial.outbound.mtls_identity.clone() + }; + let prepared = PreparedServerTls::from_snapshot(&initial).expect("config B").expect("TLS B"); + let holder = prepared.tcp_acceptor().expect("TLS B acceptor"); + let initial_acceptor = holder.get(); + let task = spawn_reload_loop( + path.clone(), + holder.clone(), + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("register HTTP initialization snapshot") + .expect("reload enabled"); + let group = RELOAD_GROUPS + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(&path) + .and_then(Weak::upgrade) + .expect("registered reload group"); + notify_server_publication(&path); + tokio::time::timeout(Duration::from_secs(10), group.pass_completed.notified()) + .await + .expect("first reload pass completed"); + let first = startup_runtime_sources::current_outbound_tls_state().await; + notify_server_publication(&path); + tokio::time::timeout(Duration::from_secs(10), group.pass_completed.notified()) + .await + .expect("second reload pass completed"); + let second = startup_runtime_sources::current_outbound_tls_state().await; + + task.abort(); + let _ = task.await; + tokio::time::timeout(Duration::from_secs(10), async { + while RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&path) { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload group exits before restoring global state"); + startup_runtime_sources::publish_outbound_tls_state( + TlsGeneration(second.generation.0.saturating_add(1)), + &rustfs_tls_runtime::OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }, + ) + .await; + crate::startup_tls_material::replace_startup_material_files(previous_files); + + assert_eq!(first.root_ca_pem.as_deref(), Some(expected_roots.as_slice())); + assert!( + first.mtls_identity == expected_identity, + "published identity must match the initialized listener snapshot" + ); + let rotate_outbound = matches!( + rotation, + StartupRotation::Outbound | StartupRotation::RootsOnly | StartupRotation::IdentityOnly + ); + let expected_generation = startup.generation.0.saturating_add(u64::from(rotate_outbound)); + assert_eq!(first.generation.0, expected_generation); + assert_eq!(second.generation, first.generation, "unchanged material must not churn outbound clients"); + assert!( + rotate_outbound || Arc::ptr_eq(&holder.get(), &initial_acceptor), + "non-publications must retain the initial server configuration" + ); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_reconciles_initial_outbound_after_startup_rotation() { + for rotation in [ + StartupRotation::RootsOnly, + StartupRotation::IdentityOnly, + StartupRotation::Outbound, + ] { + assert_initial_outbound_reconciliation(rotation).await; + } + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_server_rotation_preserves_matching_outbound_generation() { + assert_initial_outbound_reconciliation(StartupRotation::ServerOnly).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_missing_roots_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::MissingRoots).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_partial_missing_roots_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::PartialRoots).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_missing_identity_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::MissingIdentity).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn late_listener_joins_accepted_generation_when_source_is_invalid() { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars([(ENV_TLS_RELOAD_ENABLE, Some("1")), (ENV_SERVER_MTLS_ENABLE, Some("0"))], async { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "server.example"); + let source = TlsSource::from_directory(dir.path()); + let a = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("generation A"); + let prepared_a = PreparedServerTls::from_snapshot(&a).expect("prepare A").expect("TLS A"); + let first = prepared_a.tcp_acceptor().expect("config A"); + let path = dir.path().to_str().expect("UTF8 path").to_string(); + let task_a = spawn_reload_loop( + path.clone(), + first.clone(), + a, + prepared_a, + #[cfg(feature = "http3")] + None, + ) + .expect("register A") + .expect("reload enabled"); + // No async yield between producing B and registration: the actor cannot accept B. + write_test_cert_pair(dir.path(), "server.example"); + let b = rustfs_tls_runtime::TlsMaterialSnapshot::load_from_snapshot(&source.snapshot().expect("pin B")) + .expect("generation B"); + let prepared_b = PreparedServerTls::from_snapshot(&b).expect("prepare B").expect("TLS B"); + let second = prepared_b.tcp_acceptor().expect("config B"); + assert!(!Arc::ptr_eq(&first.get(), &second.get())); + fs::write(dir.path().join(RUSTFS_TLS_CERT), "invalid certificate").expect("break source"); + let task_b = spawn_reload_loop( + path.clone(), + second.clone(), + b, + prepared_b, + #[cfg(feature = "http3")] + None, + ) + .expect("register B") + .expect("reload enabled"); + assert!( + Arc::ptr_eq(&first.get(), &second.get()), + "late listener must use the group's accepted config" + ); + task_a.abort(); + task_b.abort(); + let _ = task_a.await; + let _ = task_b.await; + tokio::time::timeout(Duration::from_secs(5), async { + while RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&path) { + tokio::task::yield_now().await; + } + }) + .await + .expect("group exits when all registrations are dropped"); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn oidc_extra_ca_cert_loads_configured_bundle() { @@ -1070,15 +1795,17 @@ mod tests { ensure_rustls_crypto_provider(); let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate"); - let config = build_server_config( - ServerCertSource::SingleCert { + let config = PreparedServerTls::from_material( + Some(&RuntimeServerTlsMaterial::SingleCert { certs: vec![cert.cert.der().clone()], key: rustls::pki_types::PrivateKeyDer::try_from(cert.signing_key.serialize_der()) .expect("private key should convert"), - }, + }), None, - ServerProtocol::Http3, ) + .expect("prepare TLS") + .expect("server certificate") + .server_config(ServerProtocol::Http3) .expect("HTTP/3 TLS config should build"); assert_eq!(config.alpn_protocols, vec![b"h3".to_vec()]); diff --git a/rustfs/src/server/tls_publish.rs b/rustfs/src/server/tls_publish.rs new file mode 100644 index 000000000..ab11787e9 --- /dev/null +++ b/rustfs/src/server/tls_publish.rs @@ -0,0 +1,1341 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Durable server-leaf publication. Trust and client identity remain owned by their original source. + +use super::tls_material; +use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_ENABLE, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustfs_tls_runtime::{ServerTlsMaterial, TlsMaterialSnapshot, TlsSource}; +use rustls::pki_types::{CertificateDer, pem::PemObject}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Component, Path, PathBuf}, +}; +use uuid::Uuid; + +pub(crate) const ENV_TLS_PUBLISH_DIR: &str = "RUSTFS_TLS_PUBLISH_DIR"; +pub(crate) const SCHEMA: &str = "rustfs.tls-publish/v1alpha1"; +pub(crate) const RESOURCE: &str = "server-default"; +const MAX_CERT: usize = 64 * 1024; +const MAX_KEY: usize = 32 * 1024; +const MANIFEST: &str = "manifest.json"; +const CURRENT: &str = "server-current"; +const INITIALIZED: &str = "initialized"; +const INITIALIZED_CONTENT: &[u8] = b"rustfs.tls-publish/v1alpha1\n"; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct OperationReads { + certificates: usize, + keys: usize, + manifests: usize, + manifest_decodes: usize, + certificate_pem_traversals: usize, + leaf_der_decodes: usize, +} + +#[cfg(test)] +thread_local! { + static OPERATION_READS: std::cell::Cell = const { std::cell::Cell::new(OperationReads { + certificates: 0, keys: 0, manifests: 0, manifest_decodes: 0, + certificate_pem_traversals: 0, leaf_der_decodes: 0, + }) }; +} + +#[cfg(test)] +fn count_operation(action: impl FnOnce(&mut OperationReads)) { + OPERATION_READS.with(|counts| { + let mut value = counts.get(); + action(&mut value); + counts.set(value); + }); +} + +fn certificate_pem_items( + bytes: &[u8], +) -> impl Iterator, rustls::pki_types::pem::Error>> + '_ { + #[cfg(test)] + count_operation(|counts| counts.certificate_pem_traversals += 1); + CertificateDer::pem_slice_iter(bytes) +} + +#[cfg(test)] +#[derive(Clone, Copy)] +enum FailurePoint { + None, + AfterPointerRename, + AfterPointerRenameUnreadable, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PublishRequest { + pub schema_version: String, + pub resource_id: String, + pub expected_target_revision: String, + pub issuance_id: Uuid, + pub certificate_pem: String, + pub private_key_pem: String, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + schema_version: String, + issuance_id: Uuid, + target_revision: Uuid, + certificate_fingerprint: String, + not_after_unix: i64, + payload_digest: String, +} + +// A disk operation pins one immutable generation and verifies its envelope once. +// This value is never retained across requests or reload ticks. +struct PublishedGeneration { + manifest: Manifest, + chain: Vec>, + leaf: LeafMetadata, + key_pem: Vec, +} + +struct LeafMetadata { + fingerprint: String, + not_after_unix: i64, + names: std::collections::BTreeSet, + is_ca: bool, +} + +#[derive(Serialize)] +pub(crate) struct CertificateState { + pub schema_version: &'static str, + pub resource_id: &'static str, + pub target_revision: String, + pub certificate_fingerprint: String, + pub not_after_unix: i64, + pub last_issuance_id: Option, + pub observed_certificate_fingerprint: Option, +} + +#[derive(Serialize)] +pub(crate) struct PublishReceipt { + pub schema_version: &'static str, + pub resource_id: &'static str, + pub issuance_id: Uuid, + pub target_revision: String, + pub certificate_fingerprint: String, + pub not_after_unix: i64, + pub observed_certificate_fingerprint: Option, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PublishError { + #[error("certificate publication is disabled or not configured safely")] + Disabled, + #[error("invalid certificate publication request or source")] + Invalid, + #[error("certificate publication revision conflict")] + Conflict, + #[error("certificate publication is busy; retry the identical issuance")] + Busy, + #[error("certificate publication storage failed; retry the identical issuance")] + Storage, +} + +impl From for PublishError { + fn from(_: std::io::Error) -> Self { + Self::Storage + } +} + +fn hex_digest(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn payload_digest(certificate: &[u8], key: &[u8]) -> String { + let mut digest = Sha256::new(); + // Both inputs have bounded sizes; the fixed-width encoding survives architecture changes. + digest.update((certificate.len() as u128).to_be_bytes()); + digest.update(certificate); + digest.update((key.len() as u128).to_be_bytes()); + digest.update(key); + digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn parse_leaf(leaf: &[u8]) -> Result { + #[cfg(test)] + count_operation(|counts| counts.leaf_der_decodes += 1); + let (remaining, parsed) = x509_parser::parse_x509_certificate(leaf).map_err(|_| PublishError::Invalid)?; + if !remaining.is_empty() { + return Err(PublishError::Invalid); + } + let mut names = std::collections::BTreeSet::new(); + if let Some(san) = parsed.subject_alternative_name().map_err(|_| PublishError::Invalid)? { + for name in &san.value.general_names { + match name { + x509_parser::extensions::GeneralName::DNSName(name) => { + names.insert(format!("dns:{}", name.to_ascii_lowercase())); + } + x509_parser::extensions::GeneralName::IPAddress(bytes) => { + names.insert(format!("ip:{bytes:?}")); + } + _ => {} + } + } + } + Ok(LeafMetadata { + fingerprint: hex_digest(leaf), + not_after_unix: parsed.validity().not_after.timestamp(), + names, + is_ca: parsed + .basic_constraints() + .map_err(|_| PublishError::Invalid)? + .is_some_and(|constraints| constraints.value.ca), + }) +} + +fn snapshot_leaf(snapshot: &TlsMaterialSnapshot) -> Option { + match &snapshot.server { + Some(ServerTlsMaterial::SingleCert { certs, .. }) => certs.first().map(|cert| hex_digest(cert.as_ref())), + _ => None, + } +} + +fn observed(tls_path: &str, expected_chain: &[CertificateDer<'_>]) -> Option { + let snapshot = tls_material::observed_server_snapshot(tls_path)?; + match &snapshot.server { + Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == expected_chain => snapshot_leaf(&snapshot), + _ => None, + } +} + +fn open_regular(path: &Path, writable: bool, create: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true).write(writable).create(create); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(PublishError::Invalid); + } + Ok(file) +} + +fn read_limited(path: &Path, limit: usize) -> Result, PublishError> { + let file = open_regular(path, false, false)?; + #[cfg(test)] + count_operation(|counts| match path.file_name().and_then(|name| name.to_str()) { + Some(RUSTFS_TLS_CERT) => counts.certificates += 1, + Some(RUSTFS_TLS_KEY) => counts.keys += 1, + Some(MANIFEST) => counts.manifests += 1, + _ => {} + }); + let mut bytes = Vec::new(); + file.take(u64::try_from(limit).map_err(|_| PublishError::Invalid)? + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > limit { + return Err(PublishError::Invalid); + } + Ok(bytes) +} + +fn validate_root(root: &Path) -> Result<(), PublishError> { + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PublishError::Disabled); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.mode() & 0o077 != 0 || metadata.uid() != rustix::process::geteuid().as_raw() { + return Err(PublishError::Disabled); + } + } + #[cfg(not(unix))] + { + Err(PublishError::Disabled) + } + #[cfg(unix)] + { + Ok(()) + } +} + +fn configured_root() -> Result, PublishError> { + let root = rustfs_utils::get_env_opt_str(ENV_TLS_PUBLISH_DIR).map(PathBuf::from); + if let Some(root) = &root { + validate_root(root)?; + } + Ok(root) +} + +fn single_default_source(tls_path: &str) -> Result<(), PublishError> { + if fs::symlink_metadata(Path::new(tls_path).join("..data")).is_ok() { + return Err(PublishError::Disabled); + } + for entry in fs::read_dir(tls_path)? { + let entry = entry?; + if !entry.file_name().to_string_lossy().starts_with('.') && entry.path().is_dir() { + return Err(PublishError::Invalid); + } + } + Ok(()) +} + +fn current_generation(root: &Path) -> Result, PublishError> { + let initialized = match fs::symlink_metadata(root.join(INITIALIZED)) { + Ok(_) => { + if read_limited(&root.join(INITIALIZED), 128)? != INITIALIZED_CONTENT { + return Err(PublishError::Invalid); + } + true + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => return Err(error.into()), + }; + let pointer = root.join(CURRENT); + match fs::symlink_metadata(&pointer) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return if initialized { Err(PublishError::Invalid) } else { Ok(None) }; + } + Err(error) => return Err(error.into()), + Ok(metadata) if !metadata.file_type().is_symlink() => return Err(PublishError::Invalid), + _ => {} + } + let target = fs::read_link(pointer)?; + let mut components = target.components(); + if components.next() != Some(Component::Normal("generations".as_ref())) { + return Err(PublishError::Invalid); + } + let Some(Component::Normal(version)) = components.next() else { + return Err(PublishError::Invalid); + }; + let id = version + .to_str() + .and_then(|text| Uuid::parse_str(text).ok()) + .ok_or(PublishError::Invalid)?; + if components.next().is_some() || version != id.to_string().as_str() { + return Err(PublishError::Invalid); + } + let generations = root.join("generations"); + if fs::symlink_metadata(&generations)?.file_type().is_symlink() { + return Err(PublishError::Invalid); + } + let directory = root.join(target); + if !fs::symlink_metadata(&directory)?.is_dir() { + return Err(PublishError::Invalid); + } + PublishedGeneration::load(directory, id).map(Some) +} + +fn validate_source_separation(root: &Path, tls_path: &str) -> Result<(), PublishError> { + let root = fs::canonicalize(root)?; + let source = fs::canonicalize(tls_path)?; + if root.starts_with(&source) || source.starts_with(&root) { + return Err(PublishError::Disabled); + } + Ok(()) +} + +impl PublishedGeneration { + fn into_snapshot(self, source: &rustfs_tls_runtime::TlsSourceSnapshot) -> Result { + TlsMaterialSnapshot::load_from_snapshot_with_server_identity(source, self.chain, &self.key_pem) + .map_err(|_| PublishError::Invalid) + } + + fn load(directory: PathBuf, revision: Uuid) -> Result { + let manifest_bytes = read_limited(&directory.join(MANIFEST), 4096)?; + #[cfg(test)] + count_operation(|counts| counts.manifest_decodes += 1); + let manifest: Manifest = serde_json::from_slice(&manifest_bytes).map_err(|_| PublishError::Invalid)?; + let certificate = read_limited(&directory.join(RUSTFS_TLS_CERT), MAX_CERT)?; + let key = read_limited(&directory.join(RUSTFS_TLS_KEY), MAX_KEY)?; + let chain = certificate_pem_items(&certificate) + .collect::, _>>() + .map_err(|_| PublishError::Invalid)?; + let leaf = parse_leaf(chain.first().ok_or(PublishError::Invalid)?.as_ref())?; + if manifest.schema_version != SCHEMA + || manifest.issuance_id.is_nil() + || manifest.target_revision.is_nil() + || manifest.target_revision != revision + || manifest.certificate_fingerprint != leaf.fingerprint + || manifest.not_after_unix != leaf.not_after_unix + || manifest.payload_digest != payload_digest(&certificate, &key) + { + return Err(PublishError::Invalid); + } + Ok(Self { + manifest, + chain, + leaf, + key_pem: key, + }) + } +} + +#[cfg(test)] +fn current_directory(root: &Path) -> Result, PublishError> { + current_generation(root).map(|generation| { + generation.map(|generation| root.join("generations").join(generation.manifest.target_revision.to_string())) + }) +} + +pub(crate) fn configured_server_snapshot( + tls_path: &str, + source: &rustfs_tls_runtime::TlsSourceSnapshot, +) -> Result, PublishError> { + let Some(root) = configured_root()? else { + return Ok(None); + }; + validate_source_separation(&root, tls_path)?; + single_default_source(tls_path)?; + current_generation(&root)? + .map(|generation| generation.into_snapshot(source)) + .transpose() +} + +fn state_at(root: &Path, tls_path: &str) -> Result { + let generation = current_generation(root)?; + state_from_generation(generation.as_ref(), tls_path) +} + +fn state_from_generation(generation: Option<&PublishedGeneration>, tls_path: &str) -> Result { + if let Some(generation) = generation { + return Ok(CertificateState { + schema_version: SCHEMA, + resource_id: RESOURCE, + target_revision: generation.manifest.target_revision.to_string(), + certificate_fingerprint: generation.leaf.fingerprint.clone(), + not_after_unix: generation.leaf.not_after_unix, + last_issuance_id: Some(generation.manifest.issuance_id), + observed_certificate_fingerprint: observed(tls_path, &generation.chain), + }); + } + let snapshot = tls_material::observed_server_snapshot(tls_path).ok_or(PublishError::Disabled)?; + let Some(ServerTlsMaterial::SingleCert { certs, .. }) = &snapshot.server else { + return Err(PublishError::Invalid); + }; + let leaf = parse_leaf(certs.first().ok_or(PublishError::Invalid)?.as_ref())?; + Ok(CertificateState { + schema_version: SCHEMA, + resource_id: RESOURCE, + target_revision: format!("bootstrap-{}", leaf.fingerprint), + certificate_fingerprint: leaf.fingerprint, + not_after_unix: leaf.not_after_unix, + last_issuance_id: None, + observed_certificate_fingerprint: observed(tls_path, certs), + }) +} + +pub(crate) async fn certificate_state(tls_path: String) -> Result { + tokio::task::spawn_blocking(move || { + let root = configured_root()?.ok_or(PublishError::Disabled)?; + validate_source_separation(&root, &tls_path)?; + single_default_source(&tls_path)?; + state_at(&root, &tls_path) + }) + .await + .map_err(|_| PublishError::Storage)? +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<(), PublishError> { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + Ok(()) +} + +fn receipt(state: CertificateState, issuance_id: Uuid) -> PublishReceipt { + PublishReceipt { + schema_version: SCHEMA, + resource_id: RESOURCE, + issuance_id, + target_revision: state.target_revision, + certificate_fingerprint: state.certificate_fingerprint, + not_after_unix: state.not_after_unix, + observed_certificate_fingerprint: state.observed_certificate_fingerprint, + } +} + +fn mark_initialized(root: &Path) -> Result<(), PublishError> { + let marker = root.join(INITIALIZED); + if fs::symlink_metadata(&marker).is_ok() { + if read_limited(&marker, 128)? != INITIALIZED_CONTENT { + return Err(PublishError::Invalid); + } + } else { + // Partial writes must remain unreferenced so an identical issuance can recover after I/O failure. + let temporary = root.join(format!(".initialized-{}", Uuid::new_v4())); + write_new(&temporary, INITIALIZED_CONTENT)?; + fs::rename(temporary, &marker)?; + } + File::open(root)?.sync_all()?; + Ok(()) +} + +fn publish_at(root: &Path, tls_path: &str, request: PublishRequest) -> Result { + publish_at_inner( + root, + tls_path, + request, + #[cfg(test)] + FailurePoint::None, + ) +} + +fn publish_at_inner( + root: &Path, + tls_path: &str, + request: PublishRequest, + #[cfg(test)] failure: FailurePoint, +) -> Result { + validate_root(root)?; + validate_source_separation(root, tls_path)?; + single_default_source(tls_path)?; + if request.schema_version != SCHEMA + || request.resource_id != RESOURCE + || request.issuance_id.is_nil() + || request.certificate_pem.is_empty() + || request.certificate_pem.len() > MAX_CERT + || request.private_key_pem.is_empty() + || request.private_key_pem.len() > MAX_KEY + || request.expected_target_revision.len() > 128 + { + return Err(PublishError::Invalid); + } + // OS lock is shared by processes using this local volume. Captured CAS tokens are never refreshed on retry. + let lock = open_regular(&root.join("publisher.lock"), true, true)?; + lock.try_lock().map_err(|error| match error { + std::fs::TryLockError::WouldBlock => PublishError::Busy, + std::fs::TryLockError::Error(_) => PublishError::Storage, + })?; + let current = current_generation(root)?; + let state = state_from_generation(current.as_ref(), tls_path)?; + let digest = payload_digest(request.certificate_pem.as_bytes(), request.private_key_pem.as_bytes()); + if state.last_issuance_id == Some(request.issuance_id) { + let generation = current.as_ref().ok_or(PublishError::Invalid)?; + if generation.manifest.payload_digest != digest { + return Err(PublishError::Conflict); + } + // A prior response may have been lost after rename or directory sync. + File::open(root)?.sync_all()?; + mark_initialized(root)?; + return Ok(receipt(state, request.issuance_id)); + } + if request.expected_target_revision != state.target_revision { + return Err(PublishError::Conflict); + } + let accepted = tls_material::observed_server_snapshot(tls_path).ok_or(PublishError::Disabled)?; + let Some(ServerTlsMaterial::SingleCert { certs, .. }) = &accepted.server else { + return Err(PublishError::Invalid); + }; + let old_names = if let Some(generation) = current.as_ref() { + generation.leaf.names.clone() + } else { + if snapshot_leaf(&accepted).as_ref() != Some(&state.certificate_fingerprint) { + return Err(PublishError::Conflict); + } + parse_leaf(certs.first().ok_or(PublishError::Invalid)?.as_ref())?.names + }; + let new_chain = certificate_pem_items(request.certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|_| PublishError::Invalid)?; + let new_leaf = parse_leaf(new_chain.first().ok_or(PublishError::Invalid)?.as_ref())?; + if new_leaf.is_ca || !old_names.is_subset(&new_leaf.names) { + return Err(PublishError::Invalid); + } + let generations = root.join("generations"); + if !generations.exists() { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&generations)?; + File::open(root)?.sync_all()?; + } + if !fs::symlink_metadata(&generations)?.is_dir() { + return Err(PublishError::Invalid); + } + let revision = Uuid::new_v4(); + let directory = generations.join(revision.to_string()); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&directory)?; + let mut pointer_may_reference = false; + let result = (|| { + write_new(&directory.join(RUSTFS_TLS_CERT), request.certificate_pem.as_bytes())?; + write_new(&directory.join(RUSTFS_TLS_KEY), request.private_key_pem.as_bytes())?; + let source = TlsSource::from_directory(tls_path); + let candidate = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &source.snapshot().map_err(|_| PublishError::Invalid)?, + Some(&directory), + ) + .map_err(|_| PublishError::Invalid)?; + if !matches!(&candidate.server, Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == &new_chain) { + return Err(PublishError::Invalid); + } + tls_material::validate_candidate_replacement(&accepted, &candidate).map_err(|_| PublishError::Invalid)?; + tls_material::validate_required_material_files( + &crate::startup_tls_material::required_startup_material_files(tls_path), + &candidate, + ) + .map_err(|_| PublishError::Invalid)?; + let prepared = tls_material::PreparedServerTls::from_snapshot(&candidate) + .map_err(|_| PublishError::Invalid)? + .ok_or(PublishError::Invalid)?; + prepared.tcp_acceptor().map_err(|_| PublishError::Invalid)?; + #[cfg(feature = "http3")] + prepared.http3_config().map_err(|_| PublishError::Invalid)?; + let ledger = Manifest { + schema_version: SCHEMA.into(), + issuance_id: request.issuance_id, + target_revision: revision, + certificate_fingerprint: new_leaf.fingerprint.clone(), + not_after_unix: new_leaf.not_after_unix, + payload_digest: digest, + }; + write_new( + &directory.join(MANIFEST), + &serde_json::to_vec(&ledger).map_err(|_| PublishError::Invalid)?, + )?; + File::open(&directory)?.sync_all()?; + File::open(&generations)?.sync_all()?; + let temporary = root.join(format!(".pointer-{revision}")); + #[cfg(unix)] + std::os::unix::fs::symlink(Path::new("generations").join(revision.to_string()), &temporary)?; + #[cfg(not(unix))] + let () = Err::<(), _>(PublishError::Disabled)?; + // A rename error can itself have an uncertain outcome on a failing filesystem. + pointer_may_reference = true; + fs::rename(&temporary, root.join(CURRENT))?; + #[cfg(test)] + { + if matches!(failure, FailurePoint::AfterPointerRenameUnreadable) { + fs::remove_file(root.join(CURRENT))?; + write_new(&root.join(CURRENT), b"unreadable pointer")?; + } + if !matches!(failure, FailurePoint::None) { + return Err(PublishError::Storage); + } + } + File::open(root)?.sync_all()?; + mark_initialized(root)?; + let installed = PublishedGeneration { + manifest: ledger, + chain: new_chain, + leaf: new_leaf, + key_pem: request.private_key_pem.into_bytes(), + }; + Ok(receipt(state_from_generation(Some(&installed), tls_path)?, request.issuance_id)) + })(); + // Observation errors must never convert an uncertain publication into deletion of serving material. + if result.is_err() && !pointer_may_reference { + let _ = fs::remove_dir_all(&directory); + } + result +} + +pub(crate) async fn publish(tls_path: String, request: PublishRequest) -> Result { + if !rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE) { + return Err(PublishError::Disabled); + } + if rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, rustfs_config::DEFAULT_TRUST_LEAF_CERT_AS_CA) { + return Err(PublishError::Disabled); + } + let root = configured_root()?.ok_or(PublishError::Disabled)?; + let notification_path = tls_path.clone(); + let result = tokio::task::spawn_blocking(move || publish_at(&root, &tls_path, request)) + .await + .map_err(|_| PublishError::Storage)?; + tls_material::notify_server_publication(¬ification_path); + result +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn pair(subject: &str) -> rcgen::CertifiedKey { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + rcgen::generate_simple_self_signed(vec![subject.into()]).expect("certificate") + } + + fn build_request(expected: &str, certificate: &rcgen::CertifiedKey) -> PublishRequest { + PublishRequest { + schema_version: SCHEMA.into(), + resource_id: RESOURCE.into(), + expected_target_revision: expected.into(), + issuance_id: Uuid::new_v4(), + certificate_pem: certificate.cert.pem(), + private_key_pem: certificate.signing_key.serialize_pem(), + } + } + + fn clone_request(request: &PublishRequest) -> PublishRequest { + PublishRequest { + schema_version: request.schema_version.clone(), + resource_id: request.resource_id.clone(), + expected_target_revision: request.expected_target_revision.clone(), + issuance_id: request.issuance_id, + certificate_pem: request.certificate_pem.clone(), + private_key_pem: request.private_key_pem.clone(), + } + } + + fn measure_operation(operation: impl FnOnce() -> T) -> (T, OperationReads) { + OPERATION_READS.with(|counts| counts.set(OperationReads::default())); + let result = operation(); + (result, OPERATION_READS.with(std::cell::Cell::get)) + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_publication_rejects_missing_startup_ca_source() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (rustfs_config::ENV_TRUST_SYSTEM_CA, Some("0")), + (rustfs_config::ENV_MTLS_CLIENT_CERT, None), + (rustfs_config::ENV_MTLS_CLIENT_KEY, None), + (rustfs_config::ENV_RUSTFS_EXTRA_CA_CERT, None), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let previous = crate::startup_runtime_sources::current_outbound_tls_state().await; + let previous_files = crate::startup_tls_material::replace_startup_material_files(None); + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + fs::write(base.path().join(rustfs_config::RUSTFS_PUBLIC_CERT), pair("public-ca.example").cert.pem()) + .expect("public trust"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), pair("fallback-ca.example").cert.pem()) + .expect("fallback trust"); + let path = base.path().to_str().expect("path"); + let mut config = crate::config::Config::new("127.0.0.1:0", Vec::new()); + config.tls_path = Some(path.into()); + crate::startup_tls_material::init_outbound_tls_material(&config) + .await + .expect("real startup publication"); + fs::remove_file(base.path().join(rustfs_config::RUSTFS_CA_CERT)).expect("remove startup CA before listener"); + let initial = tls_material::load_tls_material(path).await.expect("HTTP initial snapshot"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload enabled"); + let state = state_at(root.path(), path).expect("valid bootstrap CAS state"); + let result = publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))); + let installed = current_directory(root.path()).expect("read durable pointer"); + task.abort(); + let _ = task.await; + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while tls_material::observed_server_snapshot(path).is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload group exits before restoring startup state"); + crate::startup_runtime_sources::publish_outbound_tls_state( + rustfs_tls_runtime::TlsGeneration( + crate::startup_runtime_sources::current_outbound_tls_generation().saturating_add(1), + ), + &rustfs_tls_runtime::OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }, + ) + .await; + crate::startup_tls_material::replace_startup_material_files(previous_files); + assert!( + matches!(result, Err(PublishError::Invalid)), + "source loss must reject an otherwise valid renewal" + ); + assert!(installed.is_none(), "rejected trust must not acquire a durable publication pointer"); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_operation_reads_are_bounded() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let bootstrap = state_at(root.path(), path).expect("bootstrap"); + let request = build_request(&bootstrap.target_revision, &pair("server.example")); + let retry = clone_request(&request); + publish_at(root.path(), path, request).expect("first publication"); + let (state, get_counts) = measure_operation(|| state_at(root.path(), path).expect("state")); + let (_, retry_counts) = measure_operation(|| publish_at(root.path(), path, retry).expect("retry")); + let (_, reload_counts) = measure_operation(|| { + let generation = current_generation(root.path()).expect("generation").expect("published"); + generation + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .expect("runtime snapshot") + }); + let (_, publish_counts) = measure_operation(|| { + publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))) + .expect("next publication") + }); + println!("GET={get_counts:?}; RETRY={retry_counts:?}; PUBLISH={publish_counts:?}"); + println!("RELOAD={reload_counts:?}"); + assert_eq!( + reload_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + get_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + retry_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + publish_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 2, + leaf_der_decodes: 2 + } + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_preserves_trust_rejects_stale_and_recovers_lost_receipt() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("original certificate"); + fs::rename(base.path().join(RUSTFS_TLS_CERT), base.path().join("seed-certificate.pem")) + .expect("symlink seed backing"); + std::os::unix::fs::symlink("seed-certificate.pem", base.path().join(RUSTFS_TLS_CERT)) + .expect("Certbot-style seed symlink"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("original key"); + let old_ca = pair("ca-old.example"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), old_ca.cert.pem()).expect("trust file"); + let path = base.path().to_str().expect("base path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let bootstrap = state_at(root.path(), path).expect("bootstrap state"); + let candidate = pair("server.example"); + let request = build_request(&bootstrap.target_revision, &candidate); + let retry = clone_request(&request); + let mutated = clone_request(&request); + let receipt = publish_at(root.path(), path, request).expect("durable receipt"); + assert_ne!(receipt.target_revision, bootstrap.target_revision); + assert_eq!(receipt.certificate_fingerprint, hex_digest(candidate.cert.der().as_ref())); + assert_eq!( + fs::read(base.path().join(RUSTFS_TLS_CERT)).expect("base certificate"), + original.cert.pem().as_bytes() + ); + let new_ca = pair("ca-new.example"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), new_ca.cert.pem()) + .expect("independent trust rotation"); + let directory = current_directory(root.path()).expect("pointer").expect("generation"); + let snapshot = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &TlsSource::from_directory(base.path()).snapshot().expect("source"), + Some(&directory), + ) + .expect("overlay snapshot"); + assert_eq!(snapshot.outbound.root_ca_pem, new_ca.cert.pem().as_bytes()); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + let retry_receipt = publish_at(root.path(), path, retry).expect("lost receipt recovery"); + assert_eq!(retry_receipt.target_revision, receipt.target_revision); + let mut mutated = mutated; + mutated.private_key_pem = pair("server.example").signing_key.serialize_pem(); + assert!(matches!(publish_at(root.path(), path, mutated), Err(PublishError::Conflict))); + assert!(matches!( + publish_at(root.path(), path, build_request(&bootstrap.target_revision, &pair("server.example"))), + Err(PublishError::Conflict) + )); + assert_eq!( + state_at(root.path(), path).expect("accepted state").target_revision, + receipt.target_revision + ); + let expanded = rcgen::generate_simple_self_signed(vec!["server.example".into(), "extra.example".into()]) + .expect("expanded names"); + let expanded_receipt = publish_at(root.path(), path, build_request(&receipt.target_revision, &expanded)) + .expect("pending expanded generation"); + assert!( + matches!( + publish_at(root.path(), path, build_request(&expanded_receipt.target_revision, &candidate)), + Err(PublishError::Invalid) + ), + "a later publication must preserve names of the durable generation even before it is observed" + ); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let one = build_request(&expanded_receipt.target_revision, &expanded); + let two = build_request(&expanded_receipt.target_revision, &expanded); + let root_path = root.path(); + let (one, two) = std::thread::scope(|scope| { + let barrier_one = barrier.clone(); + let first = scope.spawn(move || { + barrier_one.wait(); + publish_at(root_path, path, one) + }); + let second = scope.spawn(|| { + barrier.wait(); + publish_at(root_path, path, two) + }); + (first.join().expect("first publisher"), second.join().expect("second publisher")) + }); + assert_eq!(usize::from(one.is_ok()) + usize::from(two.is_ok()), 1, "one concurrent CAS winner"); + assert!( + matches!(one, Err(PublishError::Conflict | PublishError::Busy)) + || matches!(two, Err(PublishError::Conflict | PublishError::Busy)) + ); + let latest = state_at(root.path(), path).expect("concurrent winner"); + let interrupted = build_request(&latest.target_revision, &expanded); + let retry = clone_request(&interrupted); + assert!( + matches!( + publish_at_inner(root.path(), path, interrupted, FailurePoint::AfterPointerRename), + Err(PublishError::Storage) + ), + "failure after rename has an unknown durable outcome" + ); + let recovered = + publish_at(root.path(), path, retry).expect("same issuance recovers and syncs the installed generation"); + assert_ne!(recovered.target_revision, latest.target_revision); + assert!( + current_directory(root.path()) + .expect("installed generation survives error") + .is_some() + ); + let interrupted = build_request(&recovered.target_revision, &expanded); + let before = fs::read_dir(root.path().join("generations")).expect("generations").count(); + assert!(matches!( + publish_at_inner(root.path(), path, interrupted, FailurePoint::AfterPointerRenameUnreadable), + Err(PublishError::Storage) + )); + let after = fs::read_dir(root.path().join("generations")) + .expect("generations after unreadable pointer") + .count(); + assert_eq!(after, before + 1, "an uncertain pointer must not delete the new generation"); + fs::remove_file(root.path().join(CURRENT)).expect("simulate accepted pointer loss"); + assert!( + matches!(current_directory(root.path()), Err(PublishError::Invalid)), + "an acknowledged publisher must never silently revert to bootstrap after pointer loss" + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn expired_generation_can_renew_and_next_operation_detects_corruption() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let expired_key = rcgen::KeyPair::generate().expect("expired key"); + let mut params = rcgen::CertificateParams::new(vec!["server.example".into(), "expired-required.example".into()]) + .expect("parameters"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let expired = params.self_signed(&expired_key).expect("expired certificate"); + let certificate = expired.pem(); + let key = expired_key.serialize_pem(); + let revision = Uuid::new_v4(); + let generations = root.path().join("generations"); + fs::create_dir(&generations).expect("generations"); + let directory = generations.join(revision.to_string()); + fs::create_dir(&directory).expect("generation"); + write_new(&directory.join(RUSTFS_TLS_CERT), certificate.as_bytes()).expect("expired cert"); + write_new(&directory.join(RUSTFS_TLS_KEY), key.as_bytes()).expect("expired key"); + let leaf = parse_leaf(expired.der().as_ref()).expect("expired metadata"); + let manifest = Manifest { + schema_version: SCHEMA.into(), + issuance_id: Uuid::new_v4(), + target_revision: revision, + certificate_fingerprint: leaf.fingerprint, + not_after_unix: leaf.not_after_unix, + payload_digest: payload_digest(certificate.as_bytes(), key.as_bytes()), + }; + write_new(&directory.join(MANIFEST), &serde_json::to_vec(&manifest).expect("manifest")).expect("manifest file"); + std::os::unix::fs::symlink(Path::new("generations").join(revision.to_string()), root.path().join(CURRENT)) + .expect("pointer"); + mark_initialized(root.path()).expect("initialized"); + let state = state_at(root.path(), path).expect("expired visible head remains readable"); + assert!(state.not_after_unix < 1_000_000_000); + assert!( + current_generation(root.path()) + .expect("expired generation") + .expect("present") + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .is_err(), + "reading an expired ledger must never bypass serving validity checks" + ); + assert!( + matches!( + publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))), + Err(PublishError::Invalid) + ), + "expired durable SAN coverage must survive renewal" + ); + let renewed = + rcgen::generate_simple_self_signed(vec!["server.example".into(), "expired-required.example".into()]) + .expect("renewal"); + let request = build_request(&state.target_revision, &renewed); + let retry = clone_request(&request); + let receipt = publish_at(root.path(), path, request).expect("renew expired generation"); + assert_ne!(receipt.target_revision, state.target_revision); + let generation = current_generation(root.path()).expect("published").expect("generation"); + assert_eq!( + state_at(root.path(), path).expect("first independent read").target_revision, + receipt.target_revision + ); + let directory = root + .path() + .join("generations") + .join(generation.manifest.target_revision.to_string()); + fs::write(directory.join(RUSTFS_TLS_KEY), "corrupted key").expect("corrupt after read"); + let snapshot = generation + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .expect("the pinned operation consumes its already verified material without a second file read"); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + assert!( + matches!(state_at(root.path(), path), Err(PublishError::Invalid)), + "the next operation must revalidate disk" + ); + assert!( + matches!(publish_at(root.path(), path, retry), Err(PublishError::Invalid)), + "idempotent retry must not bypass disk corruption" + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn invalid_candidate_never_changes_pointer() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("base path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let expected = state_at(root.path(), path).expect("state").target_revision; + assert!(matches!( + publish_at(root.path(), path, build_request(&expected, &pair("wrong.example"))), + Err(PublishError::Invalid) + )); + let valid = pair("server.example"); + let mut mismatch = build_request(&expected, &valid); + mismatch.private_key_pem = pair("server.example").signing_key.serialize_pem(); + assert!(matches!(publish_at(root.path(), path, mismatch), Err(PublishError::Invalid))); + let mut ca_params = rcgen::CertificateParams::new(vec!["server.example".into()]).expect("CA params"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca_cert = ca_params.self_signed(&ca_key).expect("CA certificate"); + let mut ca_candidate = build_request(&expected, &valid); + ca_candidate.certificate_pem = ca_cert.pem(); + ca_candidate.private_key_pem = ca_key.serialize_pem(); + assert!( + matches!(publish_at(root.path(), path, ca_candidate), Err(PublishError::Invalid)), + "a CA certificate must not become the serving leaf" + ); + let mut oversized = build_request(&expected, &valid); + oversized.certificate_pem = "x".repeat(MAX_CERT + 1); + assert!(matches!(publish_at(root.path(), path, oversized), Err(PublishError::Invalid))); + assert!(current_directory(root.path()).expect("pointer absent").is_none()); + assert_eq!(state_at(root.path(), path).expect("unchanged").target_revision, expected); + task.abort(); + }, + ) + .await; + } + + #[test] + fn corrupt_or_escaping_pointer_is_rejected() { + let root = tempfile::TempDir::new().expect("root"); + std::os::unix::fs::symlink("../outside", root.path().join(CURRENT)).expect("pointer"); + assert!(matches!(current_directory(root.path()), Err(PublishError::Invalid))); + fs::remove_file(root.path().join(CURRENT)).expect("remove pointer"); + fs::write(root.path().join(CURRENT), "generations/foo").expect("regular pointer"); + assert!(matches!(current_directory(root.path()), Err(PublishError::Invalid))); + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_is_observed_by_live_tls_handshakes() { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + let renewed = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (ENV_TLS_PUBLISH_DIR, root.path().to_str()), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (rustfs_config::ENV_TRUST_SYSTEM_CA, Some("0")), + (rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (rustfs_config::ENV_RUSTFS_TLS_PATH, Some("ignored-env-source")), + ], + async { + let path = base.path().to_str().expect("path"); + let initial = tls_material::load_tls_material(path).await.expect("initial snapshot"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let registration = tls_material::spawn_reload_loop( + path.into(), + holder.clone(), + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + assert_eq!( + tls_material::active_server_tls_path(), + Some(path.to_string()), + "registered CLI/listener source wins over environment" + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let address = listener.local_addr().expect("address"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (socket, _) = listener.accept().await.expect("TCP connection"); + let _tls = holder.get().accept(socket).await.expect("TLS handshake"); + } + }); + let mut roots = rustls::RootCertStore::empty(); + roots.add(original.cert.der().clone()).expect("bootstrap anchor"); + roots.add(renewed.cert.der().clone()).expect("renewal anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let connector = tokio_rustls::TlsConnector::from(std::sync::Arc::new(config)); + async fn handshake(connector: &tokio_rustls::TlsConnector, address: std::net::SocketAddr) -> String { + let socket = tokio::net::TcpStream::connect(address).await.expect("connect"); + let stream = connector + .connect(rustls::pki_types::ServerName::try_from("server.example").expect("SNI"), socket) + .await + .expect("verified handshake"); + hex_digest(stream.get_ref().1.peer_certificates().expect("peer chain")[0].as_ref()) + } + assert_eq!(handshake(&connector, address).await, hex_digest(original.cert.der().as_ref())); + let bootstrap = certificate_state(path.into()).await.expect("bootstrap state"); + let receipt = publish(path.into(), build_request(&bootstrap.target_revision, &renewed)) + .await + .expect("publish"); + let expected = hex_digest(renewed.cert.der().as_ref()); + assert_eq!(receipt.certificate_fingerprint, expected); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if certificate_state(path.into()) + .await + .expect("state") + .observed_certificate_fingerprint + == Some(expected.clone()) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("publication observed by serving group"); + assert_eq!(handshake(&connector, address).await, expected); + let mut expected_chain = vec![renewed.cert.der().clone()]; + assert_eq!(observed(path, &expected_chain), Some(expected.clone())); + expected_chain.push(original.cert.der().clone()); + assert!( + observed(path, &expected_chain).is_none(), + "matching leaf alone must not acknowledge a different serving chain" + ); + server.await.expect("TLS server"); + registration.abort(); + }, + ) + .await; + } +} diff --git a/rustfs/src/startup_runtime_sources.rs b/rustfs/src/startup_runtime_sources.rs index 30b35458f..73501a592 100644 --- a/rustfs/src/startup_runtime_sources.rs +++ b/rustfs/src/startup_runtime_sources.rs @@ -17,6 +17,7 @@ use crate::runtime_sources::current_outbound_tls_generation as runtime_current_o use crate::storage_api::startup::runtime_sources::{InstanceContext, set_global_rustfs_port}; use rustfs_kms::KmsServiceManager; use rustfs_obs::{GlobalError as ObservabilityError, OtelGuard}; +pub(crate) use rustfs_tls_runtime::load_global_outbound_tls_state as current_outbound_tls_state; use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration}; use std::sync::Arc; use tokio_util::sync::CancellationToken; diff --git a/rustfs/src/startup_tls_material.rs b/rustfs/src/startup_tls_material.rs index 9d770293e..32931c707 100644 --- a/rustfs/src/startup_tls_material.rs +++ b/rustfs/src/startup_tls_material.rs @@ -14,7 +14,9 @@ use crate::config::Config; use crate::startup_runtime_sources; +use std::collections::BTreeSet; use std::io::{Error, Result}; +use std::sync::Mutex; use tracing::{error, info}; const LOG_COMPONENT_MAIN: &str = "main"; @@ -23,6 +25,25 @@ const EVENT_TLS_OUTBOUND_INITIALIZED: &str = "tls_outbound_initialized"; const EVENT_TLS_OUTBOUND_INITIALIZATION_FAILED: &str = "tls_outbound_initialization_failed"; const TLS_STARTUP_GENERATION_CONSUMER: &str = "rustfs_server_startup"; +// Startup completes before listeners register; retain source names independently +// of the later listener snapshot, without pinning renewable certificate bytes. +static STARTUP_MATERIAL_FILES: Mutex)>> = Mutex::new(None); + +pub(crate) fn required_startup_material_files(tls_path: &str) -> BTreeSet { + STARTUP_MATERIAL_FILES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .filter(|(path, _)| path == tls_path.trim()) + .map(|(_, files)| files.clone()) + .unwrap_or_default() +} + +#[cfg(test)] +pub(crate) fn replace_startup_material_files(files: Option<(String, BTreeSet)>) -> Option<(String, BTreeSet)> { + std::mem::replace(&mut *STARTUP_MATERIAL_FILES.lock().unwrap_or_else(|error| error.into_inner()), files) +} + pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { crate::server::tls_material::validate_configured_oidc_extra_ca_cert() .await @@ -32,6 +53,8 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { match crate::server::tls_material::load_tls_material(tls_path).await { Ok(snapshot) => { publish_outbound_tls_material(&snapshot.outbound, Some(tls_path)).await; + *STARTUP_MATERIAL_FILES.lock().unwrap_or_else(|error| error.into_inner()) = + Some((tls_path.to_string(), snapshot.material_files)); } Err(err) => { error!( @@ -49,6 +72,11 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { if startup_runtime_sources::observability_metric_enabled() { startup_runtime_sources::init_tls_metrics(); } + } else { + STARTUP_MATERIAL_FILES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); } Ok(()) diff --git a/scripts/test_helm_templates.sh b/scripts/test_helm_templates.sh index 1b21e68cb..f273430b9 100755 --- a/scripts/test_helm_templates.sh +++ b/scripts/test_helm_templates.sh @@ -568,7 +568,7 @@ nondefault_statefulset=$(awk ' nondefault_local_host=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$nondefault_statefulset") nondefault_namespace=$(yq eval '.metadata.namespace' - <<<"$nondefault_statefulset") nondefault_endpoint_port=$(yq eval '.spec.template.spec.containers[0].ports[] | select(.name == "endpoint") | .containerPort' - <<<"$nondefault_statefulset") -mtls_liveness_command=$(yq eval '.spec.template.spec.containers[0].livenessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") +mtls_liveness_port=$(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$nondefault_statefulset") mtls_readiness_command=$(yq eval '.spec.template.spec.containers[0].readinessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") if [[ "$nondefault_local_host" != '$(RUSTFS_CHART_POD_NAME).tenant-rustfs-headless.object-storage.svc.cluster.corp' ]]; then echo "Combined generated topology produced an unexpected local endpoint host: $nondefault_local_host" >&2 @@ -578,9 +578,8 @@ if [[ "$nondefault_namespace" != "object-storage" || "$nondefault_endpoint_port" echo "Combined generated topology must retain its namespace and custom endpoint port" >&2 exit 1 fi -if ! grep -Fq 'https://127.0.0.1:9443/health' <<<"$mtls_liveness_command" || - grep -Fq '/health/ready' <<<"$mtls_liveness_command"; then - echo "mTLS liveness must use the process health endpoint, not readiness" >&2 +if [[ "$mtls_liveness_port" != 9443 ]]; then + echo "mTLS liveness must check the listener independently of certificate validity" >&2 exit 1 fi if ! grep -Fq 'https://127.0.0.1:9443/health/ready' <<<"$mtls_readiness_command"; then diff --git a/scripts/test_helm_tls_reload.sh b/scripts/test_helm_tls_reload.sh new file mode 100644 index 000000000..e8ad68d86 --- /dev/null +++ b/scripts/test_helm_tls_reload.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CHART_DIR="$ROOT_DIR/helm/rustfs" + +for distributed in false true; do + rendered=$(helm template tls-reload "$CHART_DIR" \ + --set secret.allowInsecureDefaults=true \ + --set mode.distributed.enabled="$distributed" \ + --set mode.standalone.enabled="$([[ "$distributed" == false ]] && echo true || echo false)" \ + --set tls.enabled=true --set tls.existingSecret=issued-tls) + workload=$(yq eval 'select(.kind == "Deployment" or .kind == "StatefulSet")' - <<<"$rendered") + mount=$(yq eval '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "tls-material")' - <<<"$workload") + [[ $(yq eval '.mountPath' - <<<"$mount") == /opt/tls ]] + [[ $(yq eval '.readOnly' - <<<"$mount") == true ]] + [[ $(yq eval 'has("subPath")' - <<<"$mount") == false ]] + keys=$(yq eval '.spec.template.spec.volumes[] | select(.name == "tls-material") | .projected.sources[].secret.items[].key' - <<<"$workload") + [[ "$keys" == $'tls.crt\ntls.key' ]] + [[ $(yq eval '.spec.template.spec.containers[0].readinessProbe.httpGet.scheme' - <<<"$workload") == HTTPS ]] + [[ $(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$workload") == 9000 ]] + config=$(yq eval 'select(.kind == "ConfigMap") | .data' - <<<"$rendered") + [[ $(yq eval '.RUSTFS_TLS_RELOAD_ENABLE' - <<<"$config") == 1 ]] + [[ $(yq eval '.RUSTFS_TLS_KEYLOG' - <<<"$config") == 0 ]] + if [[ "$distributed" == true ]]; then + [[ $(yq eval '.RUSTFS_VOLUMES' - <<<"$config") == https://* ]] + fi + + mtls=$(helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set mode.distributed.enabled="$distributed" \ + --set mode.standalone.enabled="$([[ "$distributed" == false ]] && echo true || echo false)" \ + --set mtls.enabled=true) + workload=$(yq eval 'select(.kind == "Deployment" or .kind == "StatefulSet")' - <<<"$mtls") + [[ $(yq eval '.spec.template.spec.volumes[] | select(.name == "tls-material") | .projected.sources | length' - <<<"$workload") == 2 ]] + [[ $(yq eval '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "tls-material") | has("subPath")' - <<<"$workload") == false ]] + [[ $(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$workload") == 9000 ]] +done + +if helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set tls.enabled=true --set tls.existingSecret=issued-tls --set mtls.enabled=true >/dev/null 2>&1; then + echo "Public TLS must not silently enable global client authentication" >&2 + exit 1 +fi +if helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set tls.enabled=true >/dev/null 2>&1; then + echo "Native TLS requires an explicitly configured serving Secret" >&2 + exit 1 +fi +echo "TLS Secret projection, rotation settings, topology and probes passed"