diff --git a/Cargo.lock b/Cargo.lock index 44433ab1e..91b1a5149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9686,6 +9686,7 @@ dependencies = [ name = "rustfs-common" version = "1.0.1" dependencies = [ + "arc-swap", "hotpath", "metrics", "smallvec", @@ -10957,6 +10958,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", + "x509-parser", ] [[package]] diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 3749c929d..b772c459d 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -34,6 +34,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] +arc-swap.workspace = true hotpath.workspace = true tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } tonic = { workspace = true, features = ["gzip", "deflate"] } diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index e4cd89c88..03814d454 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use arc_swap::ArcSwap; use std::collections::HashMap; -use std::sync::LazyLock; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock}; use tokio::sync::RwLock; use tonic::transport::Channel; @@ -27,6 +28,42 @@ pub static GLOBAL_ROOT_CERT: LazyLock>>> = LazyLock::new(| pub static GLOBAL_MTLS_IDENTITY: LazyLock>> = LazyLock::new(|| RwLock::new(None)); pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock = LazyLock::new(|| AtomicU64::new(0)); +#[derive(Clone, Default)] +pub struct OutboundTlsSnapshot { + pub generation: u64, + pub root_ca_pem: Option>, + pub mtls_identity: Option, +} + +static GLOBAL_OUTBOUND_TLS_STATE: LazyLock> = + LazyLock::new(|| ArcSwap::from_pointee(OutboundTlsSnapshot::default())); + +pub fn get_global_outbound_tls_snapshot() -> Arc { + GLOBAL_OUTBOUND_TLS_STATE.load_full() +} + +/// Publish identity, trust and generation together; the individual globals are legacy views. +pub async fn publish_global_outbound_tls_snapshot(snapshot: OutboundTlsSnapshot) -> u64 { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + if current.generation != 0 + && current.root_ca_pem == snapshot.root_ca_pem + && current.mtls_identity == snapshot.mtls_identity + { + return Arc::clone(current); + } + Arc::new(OutboundTlsSnapshot { + generation: snapshot.generation.max(current.generation.saturating_add(1)), + root_ca_pem: snapshot.root_ca_pem.clone(), + mtls_identity: snapshot.mtls_identity.clone(), + }) + }); + let accepted = get_global_outbound_tls_snapshot(); + *GLOBAL_ROOT_CERT.write().await = accepted.root_ca_pem.clone(); + *GLOBAL_MTLS_IDENTITY.write().await = accepted.mtls_identity.clone(); + GLOBAL_OUTBOUND_TLS_GENERATION.store(accepted.generation, Ordering::Relaxed); + accepted.generation +} + /// Log level to use when reporting cached gRPC connection eviction. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ConnectionEvictionLogLevel { @@ -85,17 +122,27 @@ pub async fn get_global_addr() -> String { /// # Arguments /// * `cert` - A vector of bytes representing the PEM-encoded root CA certificate. pub async fn set_global_root_cert(cert: Vec) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.root_ca_pem = Some(cert.clone()); + Arc::new(next) + }); *GLOBAL_ROOT_CERT.write().await = Some(cert); } /// Clear the global root CA certificate for outbound gRPC clients. pub async fn clear_global_root_cert() { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.root_ca_pem = None; + Arc::new(next) + }); *GLOBAL_ROOT_CERT.write().await = None; } /// Get the global root CA certificate for outbound gRPC clients. pub async fn get_global_root_cert() -> Option> { - GLOBAL_ROOT_CERT.read().await.clone() + get_global_outbound_tls_snapshot().root_ca_pem.clone() } /// Set the global mTLS identity (cert+key PEM) for outbound gRPC clients. @@ -105,22 +152,32 @@ pub async fn get_global_root_cert() -> Option> { /// # Arguments /// * `identity` - An optional MtlsIdentityPem struct containing the cert and key PEM. pub async fn set_global_mtls_identity(identity: Option) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.mtls_identity = identity.clone(); + Arc::new(next) + }); *GLOBAL_MTLS_IDENTITY.write().await = identity; } /// Get the global mTLS identity for outbound gRPC clients. pub async fn get_global_mtls_identity() -> Option { - GLOBAL_MTLS_IDENTITY.read().await.clone() + get_global_outbound_tls_snapshot().mtls_identity.clone() } /// Set the global outbound TLS generation. pub fn set_global_outbound_tls_generation(generation: u64) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.generation = generation; + Arc::new(next) + }); GLOBAL_OUTBOUND_TLS_GENERATION.store(generation, Ordering::Relaxed); } /// Get the global outbound TLS generation. pub fn get_global_outbound_tls_generation() -> u64 { - GLOBAL_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed) + get_global_outbound_tls_snapshot().generation } /// Evict a stale/dead connection from the global connection cache. @@ -194,8 +251,77 @@ pub async fn clear_all_connections() { /// /// When present, gRPC clients will present this identity to servers requesting/requiring mTLS. /// When absent, clients proceed with standard server-authenticated TLS. -#[derive(Clone, Debug)] +#[derive(Clone, PartialEq, Eq)] pub struct MtlsIdentityPem { pub cert_pem: Vec, pub key_pem: Vec, } + +impl std::fmt::Debug for MtlsIdentityPem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MtlsIdentityPem") + .field("cert_bytes", &self.cert_pem.len()) + .field("key_pem", &"[REDACTED]") + .finish() + } +} + +#[cfg(test)] +mod tls_snapshot_tests { + use super::*; + + #[tokio::test] + async fn outbound_snapshot_publication_is_coherent_and_skips_unchanged_material() { + let start = get_global_outbound_tls_generation().saturating_add(1); + let snapshot = OutboundTlsSnapshot { + generation: start, + root_ca_pem: Some(vec![1]), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: vec![1], + key_pem: vec![1], + }), + }; + let first = publish_global_outbound_tls_snapshot(snapshot.clone()).await; + assert_eq!( + publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: first.saturating_add(1), + ..snapshot + }) + .await, + first + ); + let writer = tokio::spawn(async move { + for tag in 2_u8..=255 { + publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: start.saturating_add(u64::from(tag)), + root_ca_pem: Some(vec![tag]), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: vec![tag], + key_pem: vec![tag], + }), + }) + .await; + tokio::task::yield_now().await; + } + }); + while !writer.is_finished() { + let state = get_global_outbound_tls_snapshot(); + let identity = state.mtls_identity.as_ref().expect("identity in complete snapshot"); + assert_eq!(state.root_ca_pem.as_ref().expect("roots in complete snapshot"), &identity.cert_pem); + assert_eq!(identity.cert_pem, identity.key_pem); + tokio::task::yield_now().await; + } + writer.await.expect("writer completes"); + } + + #[test] + fn mtls_identity_debug_redacts_private_key() { + let identity = MtlsIdentityPem { + cert_pem: b"public certificate".to_vec(), + key_pem: b"PRIVATE-KEY-MARKER".to_vec(), + }; + let debug = format!("{identity:?}"); + assert!(!debug.contains("PRIVATE-KEY-MARKER")); + assert!(debug.contains("[REDACTED]")); + } +} diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 76ae3bb36..27bd0542f 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -24,10 +24,11 @@ use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_he #[cfg(test)] use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers}; use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError}; +#[cfg(test)] use crate::runtime::sources as runtime_sources; use http::{Request as HttpRequest, Response as HttpResponse, Uri}; use rustfs_protos::{ - ChannelClass, create_new_channel, get_channel_for_class, + ChannelClass, get_channel_for_class, proto_gen::node_service::{ heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, scanner_control_service_client::ScannerControlServiceClient, @@ -66,12 +67,9 @@ pub(crate) async fn scanner_control_time_out_client( interceptor: TonicInterceptor, ) -> crate::error::Result>> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr) - .await - .map_err(|err| crate::error::Error::other(err.to_string()))?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control) + .await + .map_err(|err| crate::error::Error::other(err.to_string()))?; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor) @@ -84,10 +82,7 @@ pub async fn heal_control_time_out_client( interceptor: TonicInterceptor, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr).await?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control).await?; let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); Ok(HealControlServiceClient::with_interceptor(channel, interceptor) @@ -100,10 +95,7 @@ pub async fn tier_mutation_control_time_out_client( interceptor: TonicInterceptor, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr).await?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control).await?; let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor) @@ -123,17 +115,7 @@ pub async fn node_service_time_out_client_for_class( class: ChannelClass, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match class { - ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await { - Some(channel) => { - debug!("Using cached gRPC channel for: {}", addr); - channel - } - // No cached connection, create new one. - None => create_new_channel(addr).await?, - }, - ChannelClass::Bulk => get_channel_for_class(addr, ChannelClass::Bulk).await?, - }; + let channel = get_channel_for_class(addr, class).await?; let max_message_size = rustfs_protos::internode_rpc_max_message_size(); let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); @@ -546,6 +528,40 @@ mod tests { use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{Registry, layer::SubscriberExt}; + #[tokio::test] + async fn rpc_client_factories_do_not_reuse_https_channels_without_tls_generation() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve local address"); + let addr = format!("https://{}", listener.local_addr().expect("local address")); + drop(listener); + for factory in 0..4 { + let placeholder = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + runtime_sources::cache_test_node_channel(addr.clone(), placeholder).await; + let interceptor = TonicInterceptor::NoOp(NoOpInterceptor); + let _result = match factory { + 0 => scanner_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + 1 => heal_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + 2 => tier_mutation_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + _ => node_service_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + }; + assert!( + !runtime_sources::test_node_channel_is_cached(&addr).await, + "factory {factory} must evict an unbound HTTPS cache entry before attempting a fresh dial" + ); + } + } + #[derive(Clone)] struct EpochProofService { audience: String, diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index f121cffe5..b7db108bf 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -50,6 +50,7 @@ use s3s::dto::BucketLifecycleConfiguration; use s3s::region::Region; use tokio::sync::{OwnedRwLockReadGuard, RwLock}; use tokio_util::sync::CancellationToken; +#[cfg(test)] use tonic::transport::Channel; use uuid::Uuid; @@ -251,10 +252,6 @@ pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option { } } -pub(crate) async fn cached_node_channel(addr: &str) -> Option { - rustfs_common::cached_connection(addr).await -} - #[cfg(test)] pub(crate) async fn cache_test_node_channel(addr: String, channel: Channel) { rustfs_common::cache_connection(addr, channel).await; diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index e2c7742d5..d86952566 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -447,6 +447,10 @@ pub enum AdminAction { ServiceFreezeAdminAction, #[strum(serialize = "admin:ConfigUpdate")] ConfigUpdateAdminAction, + #[strum(serialize = "admin:TLSCertificatePublish")] + TLSCertificatePublishAdminAction, + #[strum(serialize = "admin:TLSCertificateStatus")] + TLSCertificateStatusAdminAction, #[strum(serialize = "admin:CreateUser")] CreateUserAdminAction, #[strum(serialize = "admin:DeleteUser")] @@ -646,6 +650,8 @@ impl AdminAction { | AdminAction::ServiceStopAdminAction | AdminAction::ServiceFreezeAdminAction | AdminAction::ConfigUpdateAdminAction + | AdminAction::TLSCertificatePublishAdminAction + | AdminAction::TLSCertificateStatusAdminAction | AdminAction::CreateUserAdminAction | AdminAction::DeleteUserAdminAction | AdminAction::ListUsersAdminAction @@ -941,6 +947,34 @@ mod tests { assert!(!Action::AdminAction(get_action).is_match(&Action::AdminAction(set_action))); } + #[test] + fn tls_certificate_actions_preserve_least_privilege() { + let publish = Action::try_from("admin:TLSCertificatePublish").expect("parse certificate publish action"); + let status = Action::try_from("admin:TLSCertificateStatus").expect("parse certificate status action"); + for (action, name) in [ + (&publish, "admin:TLSCertificatePublish"), + (&status, "admin:TLSCertificateStatus"), + ] { + let Action::AdminAction(admin_action) = action else { + panic!("certificate policy must use an admin action"); + }; + assert!(admin_action.is_valid()); + assert_eq!(serde_json::to_string(action).expect("serialize action"), format!("\"{name}\"")); + } + assert!(!status.is_match(&publish)); + assert!(!publish.is_match(&status)); + let config = Action::AdminAction(AdminAction::ConfigUpdateAdminAction); + let server_info = Action::AdminAction(AdminAction::ServerInfoAdminAction); + assert!(!config.is_match(&publish)); + assert!(!server_info.is_match(&publish)); + let admin = Action::AdminAction(AdminAction::AllAdminActions); + assert!(admin.is_match(&publish)); + assert!(admin.is_match(&status)); + let s3 = Action::try_from("s3:*").expect("parse S3 wildcard"); + assert!(!s3.is_match(&publish)); + assert!(!s3.is_match(&status)); + } + #[test] fn test_table_catalog_admin_action_is_valid() { let get_action = AdminAction::try_from("admin:GetTableCatalog").expect("Should parse GetTableCatalog action"); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index a521c3a9f..451afa21a 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -25,8 +25,8 @@ use rustfs_common::{cache_connection, cached_connection, evict_connection_with_l use std::{ collections::HashMap, error::Error, - sync::LazyLock, sync::atomic::{AtomicU8, AtomicUsize, Ordering}, + sync::{Arc, LazyLock, Mutex as StdMutex, Weak}, time::{Duration, Instant}, }; use tokio::sync::Mutex; @@ -59,6 +59,44 @@ const INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET: u8 = 0; const INTERNODE_RPC_MSGPACK_ONLY_CACHE_FALSE: u8 = 1; const INTERNODE_RPC_MSGPACK_ONLY_CACHE_TRUE: u8 = 2; static TLS_GENERATION_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +// Lock order: generation metadata, then the common connection cache. The dial +// registry's standard mutex is released before waiting on any async lock. +static DIAL_LOCK_REGISTRY: LazyLock>>>> = LazyLock::new(|| StdMutex::new(HashMap::new())); + +struct DialLock { + key: String, + mutex: Arc>, +} + +impl DialLock { + fn for_key(key: &str) -> Self { + let mut registry = DIAL_LOCK_REGISTRY.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + registry.retain(|_, lock| lock.strong_count() != 0); + let mutex = registry.get(key).and_then(Weak::upgrade).unwrap_or_else(|| { + let mutex = Arc::new(Mutex::new(())); + registry.insert(key.to_string(), Arc::downgrade(&mutex)); + mutex + }); + Self { + key: key.to_string(), + mutex, + } + } +} + +impl Drop for DialLock { + fn drop(&mut self) { + let mut registry = DIAL_LOCK_REGISTRY.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if Arc::strong_count(&self.mutex) == 1 + && registry + .get(&self.key) + .is_some_and(|slot| Weak::ptr_eq(slot, &Arc::downgrade(&self.mutex))) + { + registry.remove(&self.key); + } + } +} + static INTERNODE_RPC_MSGPACK_ONLY_CACHE: AtomicU8 = AtomicU8::new(INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET); fn enforce_tls_generation_cache_bound(generation_cache: &mut HashMap, generation: u64, addr: &str) { @@ -2805,6 +2843,53 @@ fn bulk_cache_key(addr: &str, idx: usize) -> String { format!("{addr}\u{0}bulk\u{0}{idx}") } +async fn cached_channel_for_tls_generation(dial_addr: &str, cache_key: &str) -> Option { + if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) { + return cached_connection(cache_key).await; + } + let mut generations = TLS_GENERATION_CACHE.lock().await; + let Some(channel) = cached_connection(cache_key).await else { + generations.remove(cache_key); + return None; + }; + if generations.get(cache_key).copied() == Some(runtime_sources::outbound_tls_generation()) { + return Some(channel); + } + evict_connection_with_log_level(cache_key, ConnectionEvictionLogLevel::Debug).await; + generations.remove(cache_key); + runtime_sources::record_stale_grpc_channel_tls_generation(); + None +} + +async fn cache_dialed_channel(dial_addr: &str, cache_key: &str, channel: &Channel, generation: Option) -> bool { + if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) { + cache_connection(cache_key.to_string(), channel.clone()).await; + return true; + } + let mut generations = TLS_GENERATION_CACHE.lock().await; + if generation != Some(runtime_sources::outbound_tls_generation()) { + runtime_sources::record_stale_grpc_channel_tls_generation(); + return false; + } + cache_connection(cache_key.to_string(), channel.clone()).await; + // Publication can advance while the common cache's async lock is awaited. + if generation != Some(runtime_sources::outbound_tls_generation()) { + evict_connection_with_log_level(cache_key, ConnectionEvictionLogLevel::Debug).await; + generations.remove(cache_key); + runtime_sources::record_stale_grpc_channel_tls_generation(); + return false; + } + if let Some(generation) = generation { + enforce_tls_generation_cache_bound(&mut generations, generation, cache_key); + generations.insert(cache_key.to_string(), generation); + } + true +} + +fn channel_acquisition_timeout(_: tokio::time::error::Elapsed) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::TimedOut, "internode channel acquisition deadline elapsed") +} + /// Acquire a cached-or-newly-dialed channel for the given peer and channel class. /// /// For [`ChannelClass::Control`] (or whenever isolation is disabled) this is exactly the legacy @@ -2813,22 +2898,27 @@ fn bulk_cache_key(addr: &str, idx: usize) -> String { /// per-peer bulk pool and dialed on demand, physically isolating large transfers from /// control-plane RPCs. pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result> { - if class == ChannelClass::Control || !channel_isolation_enabled() { - if let Some(channel) = cached_connection(addr).await { - debug!("Using cached control gRPC channel for: {}", addr); + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + tokio::time::timeout_at(deadline, async { + if class == ChannelClass::Control || !channel_isolation_enabled() { + if let Some(channel) = cached_channel_for_tls_generation(addr, addr).await { + debug!("Using cached control gRPC channel for: {}", addr); + return Ok(channel); + } + return build_channel_until(addr, addr, deadline).await; + } + + let pool_size = bulk_channel_pool_size(); + let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size; + let cache_key = bulk_cache_key(addr, idx); + if let Some(channel) = cached_channel_for_tls_generation(addr, &cache_key).await { + debug!("Using cached bulk gRPC channel {} for: {}", idx, addr); return Ok(channel); } - return create_new_channel(addr).await; - } - - let pool_size = bulk_channel_pool_size(); - let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size; - let cache_key = bulk_cache_key(addr, idx); - if let Some(channel) = cached_connection(&cache_key).await { - debug!("Using cached bulk gRPC channel {} for: {}", idx, addr); - return Ok(channel); - } - build_channel(addr, &cache_key).await + build_channel_until(addr, &cache_key, deadline).await + }) + .await + .map_err(channel_acquisition_timeout)? } /// Creates a new gRPC channel with optimized keepalive settings for cluster resilience. @@ -2852,6 +2942,45 @@ pub async fn create_new_channel(addr: &str) -> Result> { /// for isolated bulk channels (see [`bulk_cache_key`]), letting several physically distinct /// channels to the same peer be cached independently. async fn build_channel(dial_addr: &str, cache_key: &str) -> Result> { + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + build_channel_until(dial_addr, cache_key, deadline).await +} + +async fn build_channel_until( + dial_addr: &str, + cache_key: &str, + deadline: tokio::time::Instant, +) -> Result> { + tokio::time::timeout_at(deadline, async { + if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { + return Ok(channel); + } + let slot = DialLock::for_key(cache_key); + let _guard = slot.mutex.lock().await; + loop { + // A successful concurrent miss has already populated this exact key. + if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { + return Ok(channel); + } + let (channel, generation) = dial_channel(dial_addr, cache_key).await?; + if cache_dialed_channel(dial_addr, cache_key, &channel, generation).await + && let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await + { + debug!( + "Successfully created and cached gRPC channel to: {} (cache key: {})", + dial_addr, cache_key + ); + return Ok(channel); + } + // A generation published during dial cannot be returned; rebuilding + // shares the original deadline with lock wait and all earlier dials. + } + }) + .await + .map_err(channel_acquisition_timeout)? +} + +async fn dial_channel(dial_addr: &str, cache_key: &str) -> Result<(Channel, Option), Box> { debug!("Creating new gRPC channel to: {} (cache key: {})", dial_addr, cache_key); let dial_started_at = Instant::now(); let connect_timeout = internode_connect_timeout(); @@ -2887,18 +3016,10 @@ async fn build_channel(dial_addr: &str, cache_key: &str) -> Result Result = tokio::sync::Mutex::const_new(()); + static INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK: Mutex<()> = Mutex::new(()); #[derive(Clone, PartialEq, prost::Message)] @@ -3144,6 +3253,243 @@ mod tests { assert_eq!(manifest_fields, proto_bin_json_fields("Response")); } + #[tokio::test] + async fn tls02_stale_https_control_and_bulk_cache_entries_are_evicted() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-stale-cache.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + let stale_generation = generation.checked_add(1).unwrap_or(0); + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.insert(key.clone(), stale_generation); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_none(), + "TLS02_INVARIANT stale HTTPS cache entry cannot be returned" + ); + assert!( + !rustfs_common::has_cached_connection(&key).await, + "TLS02_INVARIANT stale entry is removed from the real connection cache" + ); + } + } + + #[tokio::test] + async fn tls02_missing_https_generation_control_and_bulk_cache_entries_are_evicted() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-missing-generation.invalid:9000"; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.remove(&key); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_none(), + "TLS02_INVARIANT HTTPS channel without a generation cannot be returned" + ); + assert!( + !rustfs_common::has_cached_connection(&key).await, + "TLS02_INVARIANT unbound entry is removed from the real connection cache" + ); + } + } + + #[tokio::test] + async fn tls02_current_https_control_and_bulk_cache_entries_are_reused() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-current-generation.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.insert(key.clone(), generation); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTPS channel at the canonical generation is reused" + ); + assert!(rustfs_common::has_cached_connection(&key).await); + evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await; + TLS_GENERATION_CACHE.lock().await.remove(&key); + } + } + + #[tokio::test] + async fn tls02_http_control_and_bulk_cache_ignore_tls_generation() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "http://tls02-http-cache.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE + .lock() + .await + .insert(key.clone(), generation.checked_add(1).unwrap_or(0)); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTP cache is independent of TLS generations" + ); + TLS_GENERATION_CACHE.lock().await.remove(&key); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTP cache needs no TLS generation metadata" + ); + evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await; + } + } + + #[tokio::test] + async fn tls02_captured_identity_is_immutable_and_late_dial_keeps_its_actual_generation() { + use rustfs_common::MtlsIdentityPem; + use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration, publish_global_outbound_tls_state}; + let _test_guard = TLS02_TEST_LOCK.lock().await; + let previous = runtime_sources::outbound_tls_state().await; + let first_generation = previous.generation.0.checked_add(1).expect("test generation capacity"); + let first = OutboundTlsMaterial { + root_ca_pem: b"root-a".to_vec(), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: b"certificate-a".to_vec(), + key_pem: b"test-key-a".to_vec(), + }), + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation), &first).await; + let captured = runtime_sources::outbound_tls_state().await; + let second = OutboundTlsMaterial { + root_ca_pem: b"root-b".to_vec(), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: b"certificate-b".to_vec(), + key_pem: b"test-key-b".to_vec(), + }), + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation + 1), &second).await; + let current = runtime_sources::outbound_tls_state().await; + let addr = "https://tls02-late-dial.invalid:9000"; + let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + let cached = cache_dialed_channel(addr, addr, &channel, Some(captured.generation.0)).await; + let original_key = captured.mtls_identity.as_ref().expect("captured identity").key_pem.clone(); + let current_key = current.mtls_identity.as_ref().expect("current identity").key_pem.clone(); + let restore = OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation + 2), &restore).await; + assert_eq!( + original_key, b"test-key-a", + "TLS02_INVARIANT captured identity cannot change with publication" + ); + assert_eq!(captured.root_ca_pem.as_deref(), Some(b"root-a".as_slice())); + assert_eq!(current_key, b"test-key-b"); + assert!(!cached, "TLS02_INVARIANT late dial cannot be relabeled with the current generation"); + assert!( + !rustfs_common::has_cached_connection(addr).await, + "TLS02_INVARIANT late stale dial is never cached" + ); + assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(addr)); + } + + #[tokio::test] + async fn tls02_concurrent_control_and_bulk_misses_merge_successful_real_dials() { + use tonic::codegen::tokio_stream::{StreamExt, wrappers::TcpListenerStream}; + let _test_guard = TLS02_TEST_LOCK.lock().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind HTTP/2 test server"); + let addr = format!("http://{}", listener.local_addr().expect("test server address")); + let accepted = Arc::new(AtomicUsize::new(0)); + let counter = accepted.clone(); + let incoming = TcpListenerStream::new(listener).map(move |stream| { + if stream.is_ok() { + counter.fetch_add(1, Ordering::SeqCst); + } + stream + }); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_routes(tonic::service::Routes::default()) + .serve_with_incoming(incoming) + .await + }); + for key in [addr.clone(), bulk_cache_key(&addr, 0)] { + let mut tasks = Vec::new(); + for _ in 0..32 { + let dial_addr = addr.clone(); + let cache_key = key.clone(); + tasks.push(tokio::spawn(async move { + build_channel(&dial_addr, &cache_key).await.map_err(|error| error.to_string()) + })); + } + for task in tasks { + let channel = task.await.expect("channel acquisition task").expect("real HTTP/2 dial"); + drop(channel); + } + assert!(rustfs_common::has_cached_connection(&key).await); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key), + "TLS02_INVARIANT completed miss groups leave no lock entries" + ); + } + // Tonic returns its channel before the server task necessarily polls the + // already-established sockets out of the listener's accept queue. + tokio::time::sleep(Duration::from_millis(20)).await; + let connections = accepted.load(Ordering::SeqCst); + evict_connection_with_log_level(&addr, ConnectionEvictionLogLevel::Debug).await; + evict_connection_with_log_level(&bulk_cache_key(&addr, 0), ConnectionEvictionLogLevel::Debug).await; + server.abort(); + let _ = server.await; + assert_eq!(connections, 2, "TLS02_INVARIANT 32 misses per control/bulk key dial once each"); + } + + #[tokio::test] + async fn tls02_concurrent_waiters_share_bounded_total_deadlines_and_release_registry() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "http://tls02-contended-dial.invalid:9000"; + let owner = DialLock::for_key(addr); + let guard = owner.mutex.lock().await; + let mut tasks = Vec::new(); + for _ in 0..8 { + tasks.push(tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_millis(40); + let result = build_channel_until(addr, addr, deadline).await; + result.err().is_some_and(|error| { + error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::TimedOut) + }) + })); + } + let completed = tokio::time::timeout(Duration::from_millis(250), async { + for task in tasks { + assert!( + task.await.expect("deadline waiter"), + "TLS02_INVARIANT wait and rebuild share a typed timeout" + ); + } + }) + .await; + drop(guard); + drop(owner); + assert!( + completed.is_ok(), + "TLS02_INVARIANT eight waiters cannot accumulate eight connect timeouts" + ); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(addr), + "TLS02_INVARIANT cancelled waiters release the lock registry" + ); + } + + #[test] + fn tls02_dial_registry_reclaims_unique_keys_and_preserves_live_lock_identity() { + for index in 0..1000 { + let key = format!("http://tls02-registry-{index}.invalid"); + let first = DialLock::for_key(&key); + let second = DialLock::for_key(&key); + assert!(Arc::ptr_eq(&first.mutex, &second.mutex), "TLS02_INVARIANT one live lock per cache key"); + drop(first); + assert!(DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key)); + drop(second); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key), + "TLS02_INVARIANT last owner removes its own weak entry" + ); + } + } + #[test] fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() { let mut cache = HashMap::new(); diff --git a/crates/protos/src/runtime_sources.rs b/crates/protos/src/runtime_sources.rs index 135e3a9f5..ac685d890 100644 --- a/crates/protos/src/runtime_sources.rs +++ b/crates/protos/src/runtime_sources.rs @@ -22,6 +22,10 @@ pub(crate) async fn outbound_tls_state() -> GlobalPublishedOutboundTlsState { load_global_outbound_tls_state().await } +pub(crate) fn outbound_tls_generation() -> u64 { + rustfs_tls_runtime::load_global_outbound_tls_generation().0 +} + pub(crate) fn record_stale_grpc_channel_tls_generation() { record_tls_consumer_stale_generation(PROTOS_GRPC_CHANNEL_TLS_CONSUMER); } diff --git a/crates/tls-runtime/Cargo.toml b/crates/tls-runtime/Cargo.toml index fb8920547..443be5ea0 100644 --- a/crates/tls-runtime/Cargo.toml +++ b/crates/tls-runtime/Cargo.toml @@ -46,6 +46,7 @@ sha2.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs", "rt", "sync", "time"] } tracing.workspace = true +x509-parser.workspace = true [dev-dependencies] rcgen.workspace = true diff --git a/crates/tls-runtime/src/certs.rs b/crates/tls-runtime/src/certs.rs index 20260768f..a79c8e9c0 100644 --- a/crates/tls-runtime/src/certs.rs +++ b/crates/tls-runtime/src/certs.rs @@ -429,6 +429,209 @@ pub fn load_all_certs_from_directory( Ok(cert_key_pairs) } +/// Strict runtime loader: a discovered identity must never disappear through +/// best-effort filtering. The diagnostic inspection API remains independent. +pub fn load_all_certs_from_directory_strict( + options: CertDirectoryLoadOptions, +) -> Result>, PrivateKeyDer<'static>)>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + options.validate()?; + let dir = &options.dir_path; + let mut pairs = HashMap::new(); + if let Some(pair) = read_cert_key_pair_strict(dir, &options.cert_filename, &options.key_filename, "default", true)? { + pairs.insert("default".to_string(), pair); + } + for entry in fs::read_dir(dir).map_err(|source| TlsRuntimeError::SourceIo { + path: dir.clone(), + source, + })? { + let entry = entry.map_err(|source| TlsRuntimeError::SourceIo { + path: dir.clone(), + source, + })?; + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_str().ok_or_else(|| TlsRuntimeError::InvalidCertificate { + name: "SNI directory".to_string(), + reason: "directory name is not UTF-8".to_string(), + })?; + if !is_discoverable_cert_domain_dir(name) { + continue; + } + let metadata = fs::metadata(&path).map_err(|source| TlsRuntimeError::SourceIo { + path: path.clone(), + source, + })?; + if !metadata.is_dir() { + continue; + } + if let Some(pair) = read_cert_key_pair_strict(&path, &options.cert_filename, &options.key_filename, name, true)? { + pairs.insert(name.to_string(), pair); + } + } + Ok(pairs) +} + +pub(crate) fn read_optional_tls_file(path: &Path) -> Result>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + match fs::symlink_metadata(path) { + Ok(_) => fs::read(path).map(Some).map_err(|source| TlsRuntimeError::SourceIo { + path: path.to_path_buf(), + source, + }), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + // A disappearing pinned generation is an error, not optional absence. + if let Some(parent) = path.parent() { + fs::read_dir(parent).map_err(|source| TlsRuntimeError::SourceIo { + path: parent.to_path_buf(), + source, + })?; + } + Ok(None) + } + Err(source) => Err(TlsRuntimeError::SourceIo { + path: path.to_path_buf(), + source, + }), + } +} + +pub(crate) fn parse_certificate_pem( + pem: &[u8], + name: &str, +) -> Result>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + let certs = CertificateDer::pem_slice_iter(pem) + .collect::, _>>() + .map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if certs.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "empty certificate PEM".to_string(), + }); + } + for cert in &certs { + let (remaining, _) = + x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if !remaining.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "trailing certificate DER".to_string(), + }); + } + } + Ok(certs) +} + +pub(crate) fn parse_private_key_pem(pem: &[u8], name: &str) -> Result, crate::error::TlsRuntimeError> { + PrivateKeyDer::from_pem_slice(pem).map_err(|error| crate::error::TlsRuntimeError::InvalidPrivateKey { + name: name.to_string(), + reason: error.to_string(), + }) +} + +pub(crate) fn read_cert_key_pair_strict( + dir: &Path, + cert_filename: &str, + key_filename: &str, + name: &str, + server: bool, +) -> Result>, PrivateKeyDer<'static>)>, crate::error::TlsRuntimeError> { + let cert = read_optional_tls_file(&dir.join(cert_filename))?; + let key = read_optional_tls_file(&dir.join(key_filename))?; + match (cert, key) { + (None, None) => Ok(None), + (Some(cert), Some(key)) => { + let certs = parse_certificate_pem(&cert, name)?; + let key = parse_private_key_pem(&key, name)?; + validate_certified_key(&certs, &key, name, server)?; + Ok(Some((certs, key))) + } + _ => Err(crate::error::TlsRuntimeError::IncompleteIdentity { name: name.to_string() }), + } +} + +/// Validate before publication, including the default certificate that bypasses +/// rustls' SNI name checks. +pub fn validate_certified_key( + certs: &[CertificateDer<'static>], + key: &PrivateKeyDer<'static>, + name: &str, + server: bool, +) -> Result { + use crate::error::TlsRuntimeError; + if certs.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "empty certificate chain".to_string(), + }); + } + for (index, cert) in certs.iter().enumerate() { + let (remaining, parsed) = + x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if !remaining.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "trailing certificate DER".to_string(), + }); + } + if index == 0 { + if !parsed.validity().is_valid() { + return Err(TlsRuntimeError::CertificateValidity { name: name.to_string() }); + } + if parsed + .extended_key_usage() + .map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })? + .is_some_and(|eku| { + if server { + !eku.value.server_auth + } else { + !eku.value.client_auth + } + }) + { + return Err(if server { + TlsRuntimeError::InvalidServerEku { name: name.to_string() } + } else { + TlsRuntimeError::InvalidClientEku { name: name.to_string() } + }); + } + } + } + let certified = CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &rustls::crypto::aws_lc_rs::default_provider()) + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + certified + .keys_match() + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + if server && name != "default" { + let mut sni = ResolvesServerCertUsingSni::new(); + sni.add(name, certified.clone()) + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + } + Ok(certified) +} + fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec>, PrivateKeyDer<'static>)> { let certs = load_certs(cert_path)?; let key = load_private_key(key_path)?; @@ -494,10 +697,7 @@ pub fn create_multi_cert_resolver( let mut default_cert = None; for (domain, (certs, key)) in cert_key_pairs { - let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?; - - let certified_key = CertifiedKey::new(certs, signing_key); + let certified_key = validate_certified_key(&certs, &key, &domain, true).map_err(io::Error::other)?; if domain == "default" { default_cert = Some(Arc::new(certified_key.clone())); } else { diff --git a/crates/tls-runtime/src/error.rs b/crates/tls-runtime/src/error.rs index ac7d50691..601624d6a 100644 --- a/crates/tls-runtime/src/error.rs +++ b/crates/tls-runtime/src/error.rs @@ -23,6 +23,38 @@ pub enum TlsRuntimeError { DirectoryNotFound { path: PathBuf }, #[error("TLS path is not a directory: {path}")] NotADirectory { path: PathBuf }, + #[error("TLS source I/O error at {path}: {source}")] + SourceIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("incomplete TLS certificate/private key pair for {name}")] + IncompleteIdentity { name: String }, + #[error("invalid TLS certificate material for {name}: {reason}")] + InvalidCertificate { name: String, reason: String }, + #[error("invalid TLS private key for {name}: {reason}")] + InvalidPrivateKey { name: String, reason: String }, + #[error("TLS certificate/private key validation failed for {name}: {source}")] + InvalidCertifiedKey { + name: String, + #[source] + source: rustls::Error, + }, + #[error("invalid TLS trust anchor in {name}: {source}")] + InvalidTrustAnchor { + name: String, + #[source] + source: rustls::Error, + }, + #[error("TLS certificate for {name} is outside its validity interval")] + CertificateValidity { name: String }, + #[error("TLS certificate for {name} does not permit server authentication")] + InvalidServerEku { name: String }, + #[error("TLS certificate for {name} does not permit client authentication")] + InvalidClientEku { name: String }, + #[error("TLS source loading task failed: {0}")] + SourceTaskJoin(#[from] tokio::task::JoinError), #[error("TLS material error: {0}")] Material(String), #[error("TLS publication error: {0}")] diff --git a/crates/tls-runtime/src/lib.rs b/crates/tls-runtime/src/lib.rs index 1d5bcf232..ae25ed86b 100644 --- a/crates/tls-runtime/src/lib.rs +++ b/crates/tls-runtime/src/lib.rs @@ -28,7 +28,8 @@ pub mod state; pub use certs::{ CertDirectoryLoadOptions, TlsCertPairInspection, TlsCertPairStatus, TlsDirectoryInspection, TlsDomainInspection, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver, inspect_cert_directory, - load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key, + load_all_certs_from_directory, load_all_certs_from_directory_strict, load_cert_bundle_der_bytes, load_certs, + load_private_key, validate_certified_key, }; pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions}; pub use coordinator::{TlsConsumer, TlsReloadCoordinator}; @@ -45,7 +46,7 @@ pub use outbound::{ load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state, }; pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop}; -pub use source::{TlsFileLayout, TlsSource, TlsSourceKind}; +pub use source::{TlsFileLayout, TlsSource, TlsSourceKind, TlsSourceSnapshot}; pub use state::OutboundOnlySnapshotArgs; pub use state::{TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeStatusSnapshot}; pub use state::{TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection}; @@ -111,6 +112,8 @@ mod tests { let coordinator = TlsReloadCoordinator::new(source.clone(), TlsReloadOptions::default()); let snapshot = TlsMaterialSnapshot { source, + server_client_ca_pem: None, + material_files: std::collections::BTreeSet::new(), server: Some(ServerTlsMaterial::MultiCert { cert_key_pairs: HashMap::new(), }), diff --git a/crates/tls-runtime/src/material.rs b/crates/tls-runtime/src/material.rs index f6b3fab99..5f8f200e5 100644 --- a/crates/tls-runtime/src/material.rs +++ b/crates/tls-runtime/src/material.rs @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory, load_certs, load_private_key}; +use crate::certs::{ + CertDirectoryLoadOptions, load_all_certs_from_directory_strict, parse_certificate_pem, parse_private_key_pem, + read_optional_tls_file, validate_certified_key, +}; use crate::error::TlsRuntimeError; use crate::fingerprint::TlsFingerprint; -use crate::source::TlsSource; +use crate::source::{TlsSource, TlsSourceSnapshot}; use rustfs_common::MtlsIdentityPem; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; -use std::collections::HashMap; -use std::io::Cursor; -use std::io::ErrorKind; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use std::collections::{BTreeSet, HashMap}; use std::path::Path; #[derive(Debug, Clone)] @@ -40,115 +41,196 @@ pub enum ServerTlsMaterial { }, } +impl ServerTlsMaterial { + /// Revalidate an accepted generation without reading the filesystem. + pub fn validate(&self) -> Result<(), TlsRuntimeError> { + match self { + Self::SingleCert { certs, key } => { + crate::certs::validate_certified_key(certs, key, "default", true)?; + } + Self::MultiCert { cert_key_pairs } => { + for (name, (certs, key)) in cert_key_pairs { + crate::certs::validate_certified_key(certs, key, name, true)?; + } + } + } + Ok(()) + } +} + +impl Clone for ServerTlsMaterial { + fn clone(&self) -> Self { + match self { + Self::SingleCert { certs, key } => Self::SingleCert { + certs: certs.clone(), + key: key.clone_key(), + }, + Self::MultiCert { cert_key_pairs } => Self::MultiCert { + cert_key_pairs: cert_key_pairs + .iter() + .map(|(name, (certs, key))| (name.clone(), (certs.clone(), key.clone_key()))) + .collect(), + }, + } + } +} + #[derive(Debug)] pub struct TlsMaterialSnapshot { + /// The resolved directory generation used by every consumer of this snapshot. pub source: TlsSource, pub server: Option, pub outbound: OutboundTlsMaterial, + pub server_client_ca_pem: Option>, + pub material_files: BTreeSet, pub fingerprint: TlsFingerprint, } impl TlsMaterialSnapshot { pub async fn load(source: &TlsSource) -> Result { - let base_dir = source.validate_directory()?.to_path_buf(); + let source = source.clone(); + tokio::task::spawn_blocking(move || Self::load_from_snapshot(&source.snapshot()?)).await? + } - let server = load_server_material(source, &base_dir)?; + /// Load all consumers' material from one previously pinned directory generation. + pub fn load_from_snapshot(snapshot: &TlsSourceSnapshot) -> Result { + Self::load_from_snapshot_with_server_directory(snapshot, None) + } - let public_ca_path = base_dir.join(&source.layout.public_ca_filename); - let client_ca_path = base_dir.join(&source.fallback_ca_filename); - let client_cert_path = base_dir.join(&source.layout.client_cert_filename); - let client_key_path = base_dir.join(&source.layout.client_key_filename); + /// Load a server-only publication without replacing the pinned trust or client identity source. + pub fn load_from_snapshot_with_server_directory( + snapshot: &TlsSourceSnapshot, + server_directory: Option<&Path>, + ) -> Result { + snapshot.validate_available()?; + let source = snapshot.resolved_source(); + let base_dir = snapshot.base_dir(); + let server = load_server_material(source, server_directory.unwrap_or(base_dir))?; + Self::load_from_snapshot_with_loaded_server(snapshot, server) + } - let public_ca_pem = tokio::fs::read(&public_ca_path).await.ok(); - let client_ca_pem = tokio::fs::read(&client_ca_path).await.ok(); - let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), client_ca_pem.as_deref()); + /// Validate an already decoded default server chain while reading trust and client identity from the pinned source. + /// Serving validity, EKU and key matching are checked here even when the publication ledger was verified earlier. + pub fn load_from_snapshot_with_server_identity( + snapshot: &TlsSourceSnapshot, + certificates: Vec>, + private_key_pem: &[u8], + ) -> Result { + snapshot.validate_available()?; + let key = parse_private_key_pem(private_key_pem, "default")?; + validate_certified_key(&certificates, &key, "default", true)?; + Self::load_from_snapshot_with_loaded_server( + snapshot, + Some(ServerTlsMaterial::SingleCert { + certs: certificates, + key, + }), + ) + } - let mtls_identity = match ( - tokio::fs::read(&client_cert_path).await.ok(), - tokio::fs::read(&client_key_path).await.ok(), - ) { - (Some(cert_pem), Some(key_pem)) => { - let mut cert_reader = Cursor::new(&cert_pem); - if CertificateDer::pem_reader_iter(&mut cert_reader).next().is_none() { - return Err(TlsRuntimeError::Material("no valid certificate in client cert PEM".to_string())); + fn load_from_snapshot_with_loaded_server( + snapshot: &TlsSourceSnapshot, + server: Option, + ) -> Result { + snapshot.validate_available()?; + let source = snapshot.resolved_source(); + let base_dir = snapshot.base_dir(); + let public_ca_pem = read_optional_tls_file(&base_dir.join(&source.layout.public_ca_filename))?; + let fallback_ca_pem = read_optional_tls_file(&base_dir.join(&source.fallback_ca_filename))?; + let client_ca_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_ca_filename))?; + let mut material_files = BTreeSet::new(); + for (name, present) in [ + (&source.layout.public_ca_filename, public_ca_pem.is_some()), + (&source.fallback_ca_filename, fallback_ca_pem.is_some()), + (&source.layout.client_ca_filename, client_ca_pem.is_some()), + ] { + if present { + material_files.insert(name.clone()); + } + } + for (name, pem) in [ + (&source.layout.public_ca_filename, &public_ca_pem), + (&source.fallback_ca_filename, &fallback_ca_pem), + (&source.layout.client_ca_filename, &client_ca_pem), + ] { + if let Some(pem) = pem { + let certs = parse_certificate_pem(pem, name)?; + let mut roots = rustls::RootCertStore::empty(); + for cert in certs { + roots.add(cert).map_err(|source| TlsRuntimeError::InvalidTrustAnchor { + name: name.clone(), + source, + })?; } - - let mut key_reader = Cursor::new(&key_pem); - PrivateKeyDer::from_pem_reader(&mut key_reader) - .map_err(|e| TlsRuntimeError::Material(format!("invalid client key PEM: {e}")))?; - + } + } + let server_client_ca_pem = client_ca_pem.or_else(|| fallback_ca_pem.clone()); + let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), fallback_ca_pem.as_deref()); + let client_cert_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_cert_filename))?; + let client_key_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_key_filename))?; + if client_cert_pem.is_some() { + material_files.insert(source.layout.client_cert_filename.clone()); + } + if client_key_pem.is_some() { + material_files.insert(source.layout.client_key_filename.clone()); + } + let mtls_identity = match (client_cert_pem, client_key_pem) { + (Some(cert_pem), Some(key_pem)) => { + let certs = parse_certificate_pem(&cert_pem, "client identity")?; + let key = parse_private_key_pem(&key_pem, "client identity")?; + validate_certified_key(&certs, &key, "client identity", false)?; Some(MtlsIdentityPem { cert_pem, key_pem }) } - _ => None, + (None, None) => None, + _ => { + return Err(TlsRuntimeError::IncompleteIdentity { + name: "client identity".to_string(), + }); + } }; - + snapshot.validate_available()?; let outbound = OutboundTlsMaterial { - root_ca_pem: root_ca_pem.clone(), + root_ca_pem, mtls_identity: mtls_identity.clone(), }; - - let server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + let mut server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + if let Some(pem) = &server_client_ca_pem { + let bytes = server_fingerprint_bytes.get_or_insert_with(Vec::new); + bytes.extend_from_slice(b"\0server-client-ca\0"); + bytes.extend_from_slice(pem); + } let fingerprint = TlsFingerprint::from_optional_bytes( server_fingerprint_bytes.as_deref(), public_ca_pem.as_deref(), - client_ca_pem.as_deref(), + fallback_ca_pem.as_deref(), mtls_identity.as_ref().map(|identity| identity.cert_pem.as_slice()), mtls_identity.as_ref().map(|identity| identity.key_pem.as_slice()), ); - Ok(Self { - source: source.clone(), + source: snapshot.resolved_source().clone(), server, outbound, + server_client_ca_pem, + material_files, fingerprint, }) } } fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result, TlsRuntimeError> { - let root_cert = base_dir.join(&source.layout.server_cert_filename); - let root_key = base_dir.join(&source.layout.server_key_filename); - - let has_root_pair = root_cert.exists() && root_key.exists(); - let cert_key_pairs = load_all_certs_from_directory( + let mut cert_key_pairs = load_all_certs_from_directory_strict( CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename) .build(), - ); - - match cert_key_pairs { - Ok(cert_key_pairs) if cert_key_pairs.len() > 1 || cert_key_pairs.keys().any(|key| key != "default") => { - Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) - } - Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => { - if let Some((certs, key)) = cert_key_pairs.get("default") { - return Ok(Some(ServerTlsMaterial::SingleCert { - certs: certs.clone(), - key: key.clone_key(), - })); - } - Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) - } - Ok(_) => Ok(None), - Err(_err) if has_root_pair => { - let root_cert_path = path_to_utf8_str(&root_cert, "root TLS certificate")?; - let root_key_path = path_to_utf8_str(&root_key, "root TLS private key")?; - let certs = load_certs(root_cert_path) - .map_err(|e| TlsRuntimeError::Material(format!("load root TLS certificate {}: {e}", root_cert.display())))?; - let key = load_private_key(root_key_path) - .map_err(|e| TlsRuntimeError::Material(format!("load root TLS private key {}: {e}", root_key.display())))?; - Ok(Some(ServerTlsMaterial::SingleCert { certs, key })) - } - Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), - Err(err) => Err(TlsRuntimeError::Material(format!( - "discover server TLS certificates under '{}': {err}", - base_dir.display() - ))), + )?; + if cert_key_pairs.is_empty() { + return Ok(None); } -} - -fn path_to_utf8_str<'a>(path: &'a Path, description: &str) -> Result<&'a str, TlsRuntimeError> { - path.to_str() - .ok_or_else(|| TlsRuntimeError::Material(format!("{description} path '{}' is not valid UTF-8", path.display()))) + if cert_key_pairs.len() == 1 { + if let Some((certs, key)) = cert_key_pairs.remove("default") { + return Ok(Some(ServerTlsMaterial::SingleCert { certs, key })); + } + } + Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) } fn combine_optional_pem(primary: Option<&[u8]>, fallback: Option<&[u8]>) -> Vec { @@ -198,3 +280,425 @@ pub(crate) fn server_material_fingerprint(material: &ServerTlsMaterial) -> TlsFi let bytes = serialize_server_material_for_fingerprint(material); TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None) } + +#[cfg(test)] +mod tests { + use super::*; + use rcgen::{CertificateParams, ExtendedKeyUsagePurpose, KeyPair}; + use std::fs; + use tempfile::TempDir; + + fn write_pair(dir: &Path, domain: &str) { + let pair = rcgen::generate_simple_self_signed(vec![domain.to_string()]).expect("generate certificate"); + fs::write(dir.join("rustfs_cert.pem"), pair.cert.pem()).expect("write certificate"); + fs::write(dir.join("rustfs_key.pem"), pair.signing_key.serialize_pem()).expect("write private key"); + } + + #[test] + fn provided_server_identity_does_not_reread_server_files_and_preserves_trust() { + let base = TempDir::new().expect("base source"); + let serving = rcgen::generate_simple_self_signed(vec!["example.com".into()]).expect("server identity"); + let trust = rcgen::generate_simple_self_signed(vec!["trust.example".into()]).expect("trust source"); + fs::write(base.path().join("rustfs_cert.pem"), "unusable server certificate").expect("poison base certificate"); + fs::write(base.path().join("rustfs_key.pem"), "unusable server key").expect("poison base key"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), trust.cert.pem()).expect("trust file"); + let source = TlsSource::from_directory(base.path()).snapshot().expect("source"); + let loaded = TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + serving.signing_key.serialize_pem().as_bytes(), + ) + .expect("provided identity"); + assert_eq!(loaded.outbound.root_ca_pem, trust.cert.pem().as_bytes()); + assert!( + matches!(loaded.server, Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == vec![serving.cert.der().clone()]) + ); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), "corrupt trust").expect("corrupt independent trust"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + serving.signing_key.serialize_pem().as_bytes() + ) + .is_err(), + "providing server material must not bypass trust validation" + ); + } + + #[test] + fn provided_server_identity_preserves_key_time_and_eku_validation() { + let base = TempDir::new().expect("base source"); + let source = TlsSource::from_directory(base.path()).snapshot().expect("source"); + let serving = rcgen::generate_simple_self_signed(vec!["example.com".into()]).expect("server identity"); + let wrong_key = KeyPair::generate().expect("wrong key"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "key matching remains mandatory" + ); + let mut expired = CertificateParams::new(vec!["example.com".into()]).expect("parameters"); + expired.not_before = rcgen::date_time_ymd(2000, 1, 1); + expired.not_after = rcgen::date_time_ymd(2001, 1, 1); + let cert = expired.self_signed(&wrong_key).expect("expired certificate"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "serving validity remains mandatory" + ); + let mut client = CertificateParams::new(vec!["example.com".into()]).expect("parameters"); + client.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let cert = client.self_signed(&wrong_key).expect("client certificate"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "server EKU remains mandatory" + ); + } + + #[tokio::test] + async fn server_overlay_survives_expired_bootstrap_without_replacing_trust() { + let base = TempDir::new().expect("base source"); + let overlay = TempDir::new().expect("server overlay"); + let key = KeyPair::generate().expect("bootstrap key"); + let mut params = CertificateParams::new(vec!["example.com".into()]).expect("bootstrap params"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let expired = params.self_signed(&key).expect("expired bootstrap"); + fs::write(base.path().join("rustfs_cert.pem"), expired.pem()).expect("bootstrap certificate"); + fs::write(base.path().join("rustfs_key.pem"), key.serialize_pem()).expect("bootstrap key"); + let ca = rcgen::generate_simple_self_signed(vec!["trust.example".into()]).expect("trust source"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), ca.cert.pem()).expect("trust file"); + write_pair(overlay.path(), "example.com"); + let source = TlsSource::from_directory(base.path()); + assert!( + TlsMaterialSnapshot::load(&source).await.is_err(), + "the expired base alone must remain invalid" + ); + let loaded = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &source.snapshot().expect("pinned base"), + Some(overlay.path()), + ) + .expect("valid overlay"); + assert_eq!(loaded.outbound.root_ca_pem, ca.cert.pem().as_bytes()); + assert!(matches!(loaded.server, Some(ServerTlsMaterial::SingleCert { .. }))); + } + + #[tokio::test] + async fn snapshot_rejects_mismatched_default_private_key() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let other = KeyPair::generate().expect("generate other key"); + fs::write(dir.path().join("rustfs_key.pem"), other.serialize_pem()).expect("replace key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_partial_sni_pair_even_with_valid_default() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create SNI directory"); + fs::write(domain.join("rustfs_cert.pem"), "invalid certificate").expect("write partial SNI pair"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_invalid_existing_ca() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join(&source.layout.public_ca_filename), "invalid CA").expect("write invalid CA"); + assert!(TlsMaterialSnapshot::load(&source).await.is_err()); + } + + #[tokio::test] + async fn snapshot_rejects_incomplete_client_identity() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join(&source.layout.client_cert_filename), "invalid client certificate").expect("write client cert"); + assert!(TlsMaterialSnapshot::load(&source).await.is_err()); + } + + #[tokio::test] + async fn snapshot_rejects_expired_server_certificate() { + let dir = TempDir::new().expect("create temp directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let cert = params.self_signed(&key).expect("generate expired certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_client_only_server_eku() { + let dir = TempDir::new().expect("create temp directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let cert = params.self_signed(&key).expect("generate client-only certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_server_only_client_eku() { + let dir = TempDir::new().expect("client identity directory"); + write_pair(dir.path(), "server.example"); + let key = KeyPair::generate().expect("client key"); + let mut params = CertificateParams::new(vec!["client.example".to_string()]).expect("client params"); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let cert = params.self_signed(&key).expect("server-only certificate"); + fs::write(dir.path().join("client_cert.pem"), cert.pem()).expect("client certificate"); + fs::write(dir.path().join("client_key.pem"), key.serialize_pem()).expect("client key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::InvalidClientEku { .. }) + )); + } + #[tokio::test] + async fn snapshot_rejects_future_server_certificate() { + let dir = TempDir::new().expect("create directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.not_before = rcgen::date_time_ymd(2100, 1, 1); + params.not_after = rcgen::date_time_ymd(2101, 1, 1); + let cert = params.self_signed(&key).expect("generate future certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::CertificateValidity { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_wrong_sni_certificate_name() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("other.example.com"); + fs::create_dir(&domain).expect("create domain directory"); + write_pair(&domain, "wrong.example.com"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_malformed_complete_sni_pair() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create domain directory"); + write_pair(&domain, "sni.example.com"); + fs::write(domain.join("rustfs_cert.pem"), "invalid certificate").expect("corrupt domain certificate"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_cert_only_root() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + fs::remove_file(dir.path().join("rustfs_key.pem")).expect("remove key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::IncompleteIdentity { .. }) + )); + } + + #[tokio::test] + async fn snapshot_fingerprint_is_stable_and_certificate_only_renewal_changes_it() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + let initial = TlsMaterialSnapshot::load(&source) + .await + .expect("load valid initial generation"); + let unchanged = TlsMaterialSnapshot::load(&source).await.expect("load unchanged generation"); + assert_eq!(initial.fingerprint, unchanged.fingerprint); + let key = KeyPair::from_pem(&fs::read_to_string(dir.path().join("rustfs_key.pem")).expect("read key")) + .expect("parse existing key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.serial_number = Some(123_u64.into()); + let renewed = params.self_signed(&key).expect("renew certificate with existing key"); + fs::write(dir.path().join("rustfs_cert.pem"), renewed.pem()).expect("write renewed certificate"); + let changed = TlsMaterialSnapshot::load(&source).await.expect("load renewed generation"); + assert_ne!(initial.fingerprint, changed.fingerprint); + } + + #[tokio::test] + async fn snapshot_server_client_ca_rotation_changes_fingerprint() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + let first_ca = rcgen::generate_simple_self_signed(vec!["first-ca".to_string()]).expect("first CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), first_ca.cert.pem()).expect("write first CA"); + let initial = TlsMaterialSnapshot::load(&source).await.expect("load first CA"); + let second_ca = rcgen::generate_simple_self_signed(vec!["second-ca".to_string()]).expect("second CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), second_ca.cert.pem()).expect("write second CA"); + let changed = TlsMaterialSnapshot::load(&source).await.expect("load second CA"); + assert_ne!(initial.fingerprint, changed.fingerprint); + assert_eq!(changed.server_client_ca_pem.as_deref(), Some(second_ca.cert.pem().as_bytes())); + } + + #[cfg(unix)] + #[test] + fn snapshot_kubernetes_switch_loads_all_material_from_pinned_generation() { + use std::os::unix::fs::symlink; + let dir = TempDir::new().expect("create projection"); + let source = TlsSource::from_directory(dir.path()); + for generation in ["..a", "..b"] { + let path = dir.path().join(generation); + fs::create_dir(&path).expect("create generation"); + write_pair(&path, "example.com"); + let cert = fs::read(path.join(&source.layout.server_cert_filename)).expect("read generated certificate"); + let key = fs::read(path.join(&source.layout.server_key_filename)).expect("read generated key"); + for filename in [ + &source.layout.public_ca_filename, + &source.fallback_ca_filename, + &source.layout.client_ca_filename, + &source.layout.client_cert_filename, + ] { + fs::write(path.join(filename), &cert).expect("write generation CA/client certificate"); + } + fs::write(path.join(&source.layout.client_key_filename), key).expect("write generation client key"); + } + symlink("..a", dir.path().join("..data")).expect("link initial generation"); + for filename in [ + &source.layout.server_cert_filename, + &source.layout.server_key_filename, + &source.layout.public_ca_filename, + &source.fallback_ca_filename, + &source.layout.client_ca_filename, + &source.layout.client_cert_filename, + &source.layout.client_key_filename, + ] { + symlink(format!("..data/{filename}"), dir.path().join(filename)).expect("link projected file"); + } + let pinned = source.snapshot().expect("pin initial generation"); + let expected = TlsMaterialSnapshot::load_from_snapshot(&pinned).expect("load initial generation"); + symlink("..b", dir.path().join("..next")).expect("link next generation"); + fs::rename(dir.path().join("..next"), dir.path().join("..data")).expect("switch projection"); + let actual = TlsMaterialSnapshot::load_from_snapshot(&pinned).expect("load pinned generation after switch"); + assert_eq!(actual.fingerprint, expected.fingerprint); + assert_eq!(actual.source.base_dir, pinned.base_dir()); + assert_eq!(actual.outbound.root_ca_pem, expected.outbound.root_ca_pem); + assert_eq!(actual.server_client_ca_pem, expected.server_client_ca_pem); + assert_eq!( + actual.outbound.mtls_identity.expect("client identity").key_pem, + expected.outbound.mtls_identity.expect("expected identity").key_pem + ); + let current = TlsMaterialSnapshot::load_from_snapshot(&source.snapshot().expect("pin new generation")) + .expect("load new generation"); + assert_ne!(actual.fingerprint, current.fingerprint); + } + + #[cfg(unix)] + #[test] + fn snapshot_rejects_disappeared_projection_source_and_dangling_optional_ca() { + use std::os::unix::fs::symlink; + let dir = TempDir::new().expect("create directory"); + let source = TlsSource::from_directory(dir.path()); + write_pair(dir.path(), "example.com"); + symlink("missing-ca", dir.path().join(&source.layout.public_ca_filename)).expect("link missing CA"); + let pinned = source.snapshot().expect("pin directory"); + assert!(matches!( + TlsMaterialSnapshot::load_from_snapshot(&pinned), + Err(TlsRuntimeError::SourceIo { .. }) + )); + fs::remove_file(dir.path().join(&source.layout.public_ca_filename)).expect("remove CA link"); + fs::remove_dir_all(dir.path()).expect("remove pinned source"); + assert!(matches!( + TlsMaterialSnapshot::load_from_snapshot(&pinned), + Err(TlsRuntimeError::SourceIo { .. }) + )); + } + #[tokio::test] + async fn snapshot_rejects_mismatched_sni_key_and_malformed_chain_member() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create SNI directory"); + write_pair(&domain, "sni.example.com"); + let source = TlsSource::from_directory(dir.path()); + let valid = TlsMaterialSnapshot::load(&source) + .await + .expect("load valid multi certificate material"); + assert!(matches!(valid.server, Some(ServerTlsMaterial::MultiCert { ref cert_key_pairs }) if cert_key_pairs.len() == 2)); + fs::write( + domain.join("rustfs_key.pem"), + KeyPair::generate().expect("generate different key").serialize_pem(), + ) + .expect("replace SNI key"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertifiedKey { .. }) + )); + write_pair(&domain, "sni.example.com"); + let cert_path = dir.path().join("rustfs_cert.pem"); + let mut cert = fs::read_to_string(&cert_path).expect("read leaf certificate"); + cert.push_str("-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n"); + fs::write(cert_path, cert).expect("append malformed chain member"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_invalid_private_key_pem_and_invalid_primary_client_ca() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join("rustfs_key.pem"), "invalid key").expect("corrupt key"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidPrivateKey { .. }) + )); + write_pair(dir.path(), "example.com"); + let valid_ca = fs::read(dir.path().join("rustfs_cert.pem")).expect("read valid certificate"); + fs::write(dir.path().join(&source.fallback_ca_filename), valid_ca).expect("write valid fallback CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), "invalid primary CA").expect("write invalid primary CA"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } +} diff --git a/crates/tls-runtime/src/outbound.rs b/crates/tls-runtime/src/outbound.rs index 11c9c5916..fac3a65cb 100644 --- a/crates/tls-runtime/src/outbound.rs +++ b/crates/tls-runtime/src/outbound.rs @@ -16,8 +16,8 @@ use crate::material::OutboundTlsMaterial; use crate::metrics::record_outbound_tls_publication; use crate::state::TlsGeneration; use rustfs_common::{ - MtlsIdentityPem, clear_global_root_cert, get_global_mtls_identity, get_global_outbound_tls_generation, get_global_root_cert, - set_global_mtls_identity, set_global_outbound_tls_generation, set_global_root_cert, + MtlsIdentityPem, OutboundTlsSnapshot, get_global_outbound_tls_generation, get_global_outbound_tls_snapshot, + publish_global_outbound_tls_snapshot, }; #[derive(Debug, Clone)] @@ -35,21 +35,21 @@ pub struct GlobalOutboundTlsStateSummary { } pub async fn publish_global_outbound_tls_state(generation: TlsGeneration, material: &OutboundTlsMaterial) { - if !material.root_ca_pem.is_empty() { - set_global_root_cert(material.root_ca_pem.clone()).await; - } else { - clear_global_root_cert().await; - } - set_global_mtls_identity(material.mtls_identity.clone()).await; - set_global_outbound_tls_generation(generation.0); - record_outbound_tls_publication(generation.0, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); + let accepted = publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: generation.0, + root_ca_pem: (!material.root_ca_pem.is_empty()).then(|| material.root_ca_pem.clone()), + mtls_identity: material.mtls_identity.clone(), + }) + .await; + record_outbound_tls_publication(accepted, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); } pub async fn load_global_outbound_tls_state() -> GlobalPublishedOutboundTlsState { + let snapshot = get_global_outbound_tls_snapshot(); GlobalPublishedOutboundTlsState { - generation: TlsGeneration(get_global_outbound_tls_generation()), - root_ca_pem: get_global_root_cert().await, - mtls_identity: get_global_mtls_identity().await, + generation: TlsGeneration(snapshot.generation), + root_ca_pem: snapshot.root_ca_pem.clone(), + mtls_identity: snapshot.mtls_identity.clone(), } } diff --git a/crates/tls-runtime/src/source.rs b/crates/tls-runtime/src/source.rs index dee0b0277..a35b68001 100644 --- a/crates/tls-runtime/src/source.rs +++ b/crates/tls-runtime/src/source.rs @@ -92,3 +92,125 @@ impl TlsSource { Ok(self.base_dir.as_path()) } } + +/// A directory generation resolved once before loading any TLS material. +/// +/// Projected Kubernetes volumes are read through their physical `..data` +/// directory. Ordinary directory and certificate-file symlinks remain supported +/// for layouts such as Certbot. Writers must keep pinned generations immutable. +#[derive(Debug, Clone)] +pub struct TlsSourceSnapshot { + source: TlsSource, + resolved_source: TlsSource, +} + +impl TlsSource { + pub fn snapshot(&self) -> Result { + self.validate_directory()?; + let mut resolved_source = self.clone(); + let root = std::fs::canonicalize(&self.base_dir).map_err(|source| TlsRuntimeError::SourceIo { + path: self.base_dir.clone(), + source, + })?; + let projection = root.join("..data"); + resolved_source.base_dir = match std::fs::symlink_metadata(&projection) { + Ok(metadata) if metadata.file_type().is_symlink() => { + std::fs::canonicalize(&projection).map_err(|source| TlsRuntimeError::SourceIo { + path: projection, + source, + })? + } + Ok(_) => root, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => root, + Err(source) => { + return Err(TlsRuntimeError::SourceIo { + path: projection, + source, + }); + } + }; + resolved_source.validate_directory()?; + Ok(TlsSourceSnapshot { + source: self.clone(), + resolved_source, + }) + } +} + +impl TlsSourceSnapshot { + pub fn source(&self) -> &TlsSource { + &self.source + } + + pub fn resolved_source(&self) -> &TlsSource { + &self.resolved_source + } + + pub fn base_dir(&self) -> &Path { + &self.resolved_source.base_dir + } + + /// Missing optional files are allowed only while the pinned source exists. + pub(crate) fn validate_available(&self) -> Result<(), TlsRuntimeError> { + std::fs::read_dir(self.base_dir()).map_err(|source| TlsRuntimeError::SourceIo { + path: self.base_dir().to_path_buf(), + source, + })?; + Ok(()) + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::fs; + use std::os::unix::fs::symlink; + use tempfile::TempDir; + + #[test] + fn source_snapshot_pins_kubernetes_projection_generation() { + let dir = TempDir::new().expect("create projection directory"); + for generation in ["..generation-a", "..generation-b"] { + fs::create_dir(dir.path().join(generation)).expect("create generation"); + fs::write(dir.path().join(generation).join("marker"), generation).expect("write generation marker"); + } + symlink("..generation-a", dir.path().join("..data")).expect("link initial generation"); + symlink("..data/marker", dir.path().join("marker")).expect("link projected file"); + let pinned = TlsSource::from_directory(dir.path()).snapshot().expect("pin projection"); + symlink("..generation-b", dir.path().join("..next-data")).expect("link next generation"); + fs::rename(dir.path().join("..next-data"), dir.path().join("..data")).expect("swap projection"); + assert_eq!( + fs::read_to_string(pinned.base_dir().join("marker")).expect("read pinned file"), + "..generation-a" + ); + assert_eq!( + fs::read_to_string(dir.path().join("marker")).expect("read current projection"), + "..generation-b" + ); + } + + #[test] + fn source_snapshot_pins_tls_path_directory_symlink() { + let dir = TempDir::new().expect("create TLS directory"); + fs::create_dir(dir.path().join("a")).expect("create generation A"); + fs::create_dir(dir.path().join("b")).expect("create generation B"); + symlink("a", dir.path().join("live")).expect("link generation A"); + let pinned = TlsSource::from_directory(dir.path().join("live")) + .snapshot() + .expect("pin TLS source"); + fs::remove_file(dir.path().join("live")).expect("unlink source"); + symlink("b", dir.path().join("live")).expect("link generation B"); + assert_eq!(pinned.base_dir(), fs::canonicalize(dir.path().join("a")).expect("resolve generation A")); + } + + #[test] + fn source_snapshot_rejects_disappeared_generation() { + let dir = TempDir::new().expect("create TLS directory"); + let generation = dir.path().join("..generation"); + fs::create_dir(&generation).expect("create generation"); + symlink("..generation", dir.path().join("..data")).expect("link generation"); + let pinned = TlsSource::from_directory(dir.path()).snapshot().expect("pin TLS source"); + fs::remove_dir(&generation).expect("remove pinned generation"); + assert!(matches!(pinned.validate_available(), Err(TlsRuntimeError::SourceIo { .. }))); + } +} diff --git a/docs/README.md b/docs/README.md index 89549fcd2..2b89a015d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,8 @@ collection: ## Operations +- [External certificate issuance and TLS rotation](operations/tls-certificate-rotation.md) — publication, native reload, trust separation and failure handling. + - [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures. - [Logging governance](operations/logging-governance.md) — broad event audits, migrations, and guardrail changes. diff --git a/docs/operations/tls-certificate-rotation.md b/docs/operations/tls-certificate-rotation.md new file mode 100644 index 000000000..750f17c6d --- /dev/null +++ b/docs/operations/tls-certificate-rotation.md @@ -0,0 +1,204 @@ +# External certificate issuance and TLS rotation + +Use an independent certificate controller to obtain and renew certificates. +RustFS consumes certificate material; it does not register ACME accounts or +change DNS records. The independent +[rustfs-cert-controller](https://github.com/houseme/rustfs-cert-controller) +owns issuance, renewal scheduling and publication. `plugin-runner` starts its +container; it does not own certificate schedules or CA credentials. Progress +and deployment acceptance are tracked in +[backlog #2806](https://github.com/rustfs/backlog/issues/2806). + +## Deployment boundaries + +For a distributed Kubernetes deployment, prefer public TLS at an +Ingress/Gateway and private TLS/mTLS between the ingress and RustFS. Public +CA certificates cannot cover internal names such as `*.svc.cluster.local`; +DNS-01 can cover names under a controlled public domain even when the service +is private. A listener +that requires mTLS also requires client certificates from browsers and S3 +clients; selecting a different SNI certificate does not change that policy. + +Native server TLS can consume an externally managed `kubernetes.io/tls` +Secret with only `tls.crt` and `tls.key`: + +```yaml +mode: + distributed: + enabled: false + standalone: + enabled: true +ingress: + enabled: false +tls: + enabled: true + existingSecret: rustfs-serving-tls + reloadInterval: 30 +``` + +Create and renew that Secret using cert-manager or an independently managed +publisher. Configure the issuing resource with the actual server DNS/IP SANs +and explicit private-key rotation. The chart does not install cert-manager, +create an ACME account or grant DNS permissions. For distributed native TLS, +the certificate must also cover the configured peer hostnames; explicitly +configure independent outbound trust when a private server CA is used. + +`tls.trustSecret` optionally names a separate Secret with an explicitly chosen +`ca.crt`, mounted as outbound `public.crt`. It does not enable mTLS or add a +public issuing CA to client authentication trust. Do not write a fabricated +`ca.crt` into the Secret managed by an ACME issuer. + +`tls.enabled` and `mtls.enabled` are mutually exclusive. Existing mTLS mode +keeps its separate server/client certificate Secrets and projects their +contents into one read-only directory. The chart enables certificate reload +for either mode and disables TLS key logging. Explicit container environment +variables can override these settings. + +## Material layout and publication + +`RUSTFS_TLS_PATH` contains `rustfs_cert.pem` and `rustfs_key.pem`. SNI +identities can reside in visible domain subdirectories with the same two +filenames. Outbound trust uses `public.crt` and `ca.crt`; inbound mTLS client +trust prefers `client_ca.crt`, falling back to `ca.crt` only at initial load. +The default outbound client identity is `client_cert.pem`/`client_key.pem`. + +Runtime loading validates existing PEM/DER, key pairing, leaf validity and +explicit authentication EKUs. Each named SNI identity must match its domain. +Chain structure validation does not replace issuer/trust-policy verification +by the issuing controller and client PKI verifier. + +Kubernetes mounts must project the entire directory; single-file `subPath` +mounts do not receive Secret updates. RustFS pins one physical `..data` +generation before loading its material. Separate controllers may update the +serving identity and trust independently, so private CA migration still needs +an old-plus-new trust overlap. + +For VMs, publish immutable version directories and atomically change the +directory pointer after syncing the complete candidate and its containing +directories. Keep current/accepted/in-use versions during cleanup. Certbot +file symlinks remain compatible, but independent file replacements do not +provide an atomic directory generation; key mismatch is rejected, and a +publisher should use a complete immutable bundle for coherent publication. + +`RUSTFS_MTLS_CLIENT_CERT`/`RUSTFS_MTLS_CLIENT_KEY` overrides are validated as +one identity. Explicit missing, malformed, mismatched, expired or unsuitable +override material rejects the update; it does not silently select another +identity. Keep default identity files valid when they are present as well. + +## Controller publication API + +For a VM or Docker installation with one default serving certificate, enable +reload and configure `RUSTFS_TLS_PUBLISH_DIR` to a dedicated persistent directory +owned by the RustFS process with mode `0700`. Keep `RUSTFS_TLS_PATH` configured +for the initial serving identity and independent trust/client material. The +publication directory must be separate from the original TLS source. A valid +initial certificate and trusted management credentials must be provisioned +before a controller can connect to the HTTPS API. That bootstrap identity must +cover the names the configured issuer will continue to renew; publication +preserves existing SAN coverage. + +The controller signs requests using the existing S3 SigV4 authentication: + +- `GET /rustfs/admin/v3/tls/certificate-state` requires + `admin:TLSCertificateStatus`. +- `POST /rustfs/admin/v3/tls/certificate` requires + `admin:TLSCertificatePublish` and a connection terminated by RustFS TLS. + +Grant only these two actions to the controller identity. Read permissions, +general configuration permissions and forwarded HTTPS headers do not authorize +private-key publication. Keep API requests and credentials out of access-body +logs. The API does not add MinIO-compatible aliases. + +The `rustfs.tls-publish/v1alpha1` request fixes `resource_id` to `server-default` +and carries an `issuance_id`, the captured `expected_target_revision`, +`certificate_pem` and `private_key_pem`. Read the target token before issuance; +retain it with that immutable candidate. Repeating the same issuance and payload +recovers a lost receipt. A conflict stops that candidate; fetching a new token +and replaying the old candidate would defeat this concurrency protection. +An HTTP 503 or uncertain network result is retryable with the same issuance, +payload and token; HTTP 409 indicates a revision or payload conflict. + +RustFS validates the complete candidate before publishing an immutable leaf +generation and syncing its atomic pointer. It does not copy or overwrite CA +trust or client credentials. This mode rejects projected Kubernetes sources, +multiple SNI identities and configurations that trust the serving leaf as a CA. +Use the existing external-file/cert-manager integration for those deployments. + +The state reports the visible target revision and public leaf fingerprint. A +successful POST receipt proves the publication was synced; a GET during an +uncertain write does not replace that receipt. Retry the identical POST to +confirm acceptance. `observed_certificate_fingerprint` is present only when +this process's serving reload group has the exact current certificate chain; +matching the leaf alone is insufficient. A durable receipt can precede that +observation. Check convergence and a fresh TLS handshake on every relevant +endpoint. One process's response does not prove another node has converged. + +Use a stable local publication volume with one owner. Captured-token CAS and +local process locking do not implement HA leadership, restore ordering or PITR +protection. Do not remove active generation directories; automated generation +cleanup is not part of this interface. + +## Activation and failure handling + +Native `RUSTFS_TLS_RELOAD_ENABLE` defaults to false. Set it to `1` outside the +chart. `RUSTFS_TLS_RELOAD_INTERVAL` defaults to 30 seconds and is clamped to a +minimum of 5 seconds. + +Listeners sharing the same configured TLS directory use one reload group. +The group prepares TCP and enabled HTTP/3 configurations from the same +snapshot before swapping them. A late listener first joins the accepted +configuration, even if the current files have become invalid. TCP, QUIC and +separate processes still have a short activation window; there is no global +atomic certificate switch. +TCP and QUIC derive their protocol settings from one prepared certificate +resolver and client verifier for that generation. Publication state and retries +load each immutable generation once per operation; the next operation validates +its disk contents again. Reload uses the verified serving identity with the +independently pinned trust/client source, without reopening its certificate and +key files. Candidate staging still performs its separate disk validation. + +Unchanged material is skipped. A serving certificate renewal does not advance +the outbound generation unless outbound trust or identity actually changes. +Outbound state is published as one immutable trust/identity/generation +snapshot. Control and bulk RPC cache acquisition checks that generation; +existing in-flight calls retain their channels. + +Reload rejects the disappearance of accepted CA/identity source components, +SNI entries or accepted DNS/IP SAN coverage. Removing or renaming an identity +or trust source requires an explicit configuration change and restart; +ordinary renewal must preserve those requirements. Rejected updates retain +the last accepted in-memory configuration. New handshakes reject certificates +outside their validity interval. Normal renewal does not terminate existing +uploads, downloads or sessions. + +The chart uses TCP listener liveness for TLS deployments so certificate expiry +does not trigger a restart loop. Readiness continues to check the TLS service. +This liveness probe detects listener availability, not complete application +health. Keep ingress traffic admission separate from internal peer routing +and storage quorum. Do not restart every node because a CA is unavailable. + +There is no automatic persisted private-key fallback in Kubernetes. On +restart, invalid explicitly configured TLS material prevents startup. Key +compromise, explicit certificate revocation, session invalidation, per-purpose +profile coordination and deployment-boundary fencing require the separate +operator/control-plane workflow; ordinary reload does not implement them. + +## Verification + +Check a fresh TLS connection to each endpoint and compare the leaf serial, +public certificate fingerprint, SANs and expiry. Include HTTP/2, enabled +HTTP/3, Console and internal RPC endpoints. Do not infer served certificates +from Secret contents or an outbound generation gauge alone. + +Focused local checks: + +```sh +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-tls-runtime +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-protos --lib +bash scripts/test_helm_tls_reload.sh +``` + +Production acceptance also requires real Kubernetes projection, CA/DNS +issuance and renewal, outage/expiry, long-transfer continuity, private CA +overlap and distributed convergence tests. These are separate from protocol +and local loader tests. diff --git a/helm/rustfs/templates/_helpers.tpl b/helm/rustfs/templates/_helpers.tpl index 126e07e4f..4c3cac02a 100644 --- a/helm/rustfs/templates/_helpers.tpl +++ b/helm/rustfs/templates/_helpers.tpl @@ -154,6 +154,10 @@ Merges (in order of increasing precedence): }} {{- $annotations = merge $annotations $mtls }} {{- end }} +{{- $tls := .Values.tls | default dict -}} +{{- if and ($tls.enabled | default false) (eq .Values.ingress.className "traefik") -}} +{{- $_ := set $annotations "traefik.ingress.kubernetes.io/service.serversscheme" "https" -}} +{{- end -}} {{- if $annotations }} {{- toYaml $annotations }} {{- end }} @@ -188,6 +192,10 @@ Merges (in order of increasing precedence): {{- end }} {{- $annotations = merge $annotations (default (dict) .Values.ingress.customAnnotations) }} {{- $annotations = merge $annotations (default (dict) .Values.ingress.annotations) }} +{{- $tls := .Values.tls | default dict -}} +{{- if and ($tls.enabled | default false) (eq .Values.ingress.className "nginx") -}} +{{- $_ := set $annotations "nginx.ingress.kubernetes.io/backend-protocol" "HTTPS" -}} +{{- end -}} {{- if $annotations }} {{- toYaml $annotations }} {{- end }} @@ -285,7 +293,8 @@ RUSTFS_VOLUMES on spaces, one pool per expression). */}} {{- define "rustfs.volumes" -}} {{- $protocol := "http" -}} -{{- if .Values.mtls.enabled -}} +{{- $tls := .Values.tls | default dict -}} +{{- if or .Values.mtls.enabled ($tls.enabled | default false) -}} {{- $protocol = "https" -}} {{- end -}} {{- $headless := printf "%s-headless" (include "rustfs.fullname" .) -}} @@ -347,17 +356,17 @@ Render liveness and readiness probe for http and https */}} {{- define "rustfs.probes" -}} +{{- $tls := .Values.tls | default dict -}} {{- if .Values.livenessProbe.enabled }} livenessProbe: - {{- if .Values.mtls.enabled }} - exec: - command: -{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health") | nindent 6 }} + {{- if or .Values.mtls.enabled ($tls.enabled | default false) }} + tcpSocket: + port: {{ .Values.service.endpoint.port | default 9000 }} {{- else }} httpGet: path: /health port: {{ .Values.service.endpoint.port | default 9000 }} - scheme: {{ if .Values.mtls.enabled }}HTTPS{{ else }}HTTP{{ end }} + scheme: {{ if or .Values.mtls.enabled ($tls.enabled | default false) }}HTTPS{{ else }}HTTP{{ end }} {{- end }} initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds | default 60 }} periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 5 }} @@ -376,7 +385,7 @@ readinessProbe: httpGet: path: /health/ready port: {{ .Values.service.endpoint.port | default 9000 }} - scheme: {{ if .Values.mtls.enabled }}HTTPS{{ else }}HTTP{{ end }} + scheme: {{ if or .Values.mtls.enabled ($tls.enabled | default false) }}HTTPS{{ else }}HTTP{{ end }} {{- end }} initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 60 }} periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 5 }} diff --git a/helm/rustfs/templates/configmap.yaml b/helm/rustfs/templates/configmap.yaml index 46e8331f8..e5aaa352f 100644 --- a/helm/rustfs/templates/configmap.yaml +++ b/helm/rustfs/templates/configmap.yaml @@ -1,3 +1,8 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} +{{- if and $tlsEnabled .Values.mtls.enabled -}} +{{- fail "tls.enabled and mtls.enabled are mutually exclusive; use ingress TLS for public access to an mTLS backend" -}} +{{- end -}} apiVersion: v1 kind: ConfigMap metadata: @@ -41,8 +46,12 @@ data: {{- end }} {{- if .Values.mtls.enabled }} RUSTFS_SERVER_MTLS_ENABLE: "1" - RUSTFS_TLS_KEYLOG: "1" + {{- end }} + {{- if or .Values.mtls.enabled $tlsEnabled }} + RUSTFS_TLS_KEYLOG: "0" RUSTFS_TLS_PATH: "/opt/tls" + RUSTFS_TLS_RELOAD_ENABLE: "1" + RUSTFS_TLS_RELOAD_INTERVAL: {{ $tls.reloadInterval | default 30 | quote }} {{- end }} {{- if .Values.config.rustfs.obs_endpoint.enabled }} RUSTFS_OBS_ENDPOINT: {{ .Values.config.rustfs.obs_endpoint.base_endpoint | quote }} diff --git a/helm/rustfs/templates/deployment.yaml b/helm/rustfs/templates/deployment.yaml index 39effeb24..97b663626 100644 --- a/helm/rustfs/templates/deployment.yaml +++ b/helm/rustfs/templates/deployment.yaml @@ -1,3 +1,5 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} @@ -123,25 +125,10 @@ spec: {{- toYaml .Values.resources | nindent 12 }} {{- include "rustfs.probes" . | nindent 10 }} volumeMounts: - {{- if .Values.mtls.enabled }} - - name: server-cert - mountPath: /opt/tls/rustfs_cert.pem - subPath: rustfs_cert.pem - - name: server-cert - mountPath: /opt/tls/rustfs_key.pem - subPath: rustfs_key.pem - - name: server-cert - mountPath: /opt/tls/ca.crt - subPath: ca.crt - - name: client-cert - mountPath: /opt/tls/client_cert.pem - subPath: client_cert.pem - - name: client-cert - mountPath: /opt/tls/client_key.pem - subPath: client_key.pem - - name: client-cert - mountPath: /opt/tls/client_ca.crt - subPath: client_ca.crt + {{- if or .Values.mtls.enabled $tlsEnabled }} + - name: tls-material + mountPath: /opt/tls + readOnly: true {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -157,27 +144,40 @@ spec: {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.mtls.enabled }} - - name: server-cert - secret: - secretName: {{ include "rustfs.fullname" . }}-server-tls - items: - - key: tls.crt - path: rustfs_cert.pem - - key: tls.key - path: rustfs_key.pem - - key: ca.crt - path: ca.crt - - name: client-cert - secret: - secretName: {{ include "rustfs.fullname" . }}-client-tls - items: - - key: tls.crt - path: client_cert.pem - - key: tls.key - path: client_key.pem - - key: ca.crt - path: client_ca.crt + {{- if or .Values.mtls.enabled $tlsEnabled }} + - name: tls-material + projected: + defaultMode: 0440 + sources: + - secret: + name: {{ if .Values.mtls.enabled }}{{ include "rustfs.fullname" . }}-server-tls{{ else }}{{ required "tls.existingSecret is required when tls.enabled=true" $tls.existingSecret }}{{ end }} + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem + {{- if .Values.mtls.enabled }} + - key: ca.crt + path: ca.crt + {{- end }} + {{- if .Values.mtls.enabled }} + - secret: + name: {{ include "rustfs.fullname" . }}-client-tls + items: + - key: tls.crt + path: client_cert.pem + - key: tls.key + path: client_key.pem + - key: ca.crt + path: client_ca.crt + {{- end }} + {{- if and $tlsEnabled $tls.trustSecret }} + - secret: + name: {{ $tls.trustSecret }} + items: + - key: ca.crt + path: public.crt + {{- end }} {{- end }} {{- if $logDirEnabled }} - name: logs diff --git a/helm/rustfs/templates/statefulset.yaml b/helm/rustfs/templates/statefulset.yaml index f2c90877d..54de97694 100644 --- a/helm/rustfs/templates/statefulset.yaml +++ b/helm/rustfs/templates/statefulset.yaml @@ -1,3 +1,5 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} {{- $poolsEnabled := and .Values.pools .Values.pools.enabled }} @@ -217,25 +219,10 @@ spec: {{- toYaml $.Values.resources | nindent 12 }} {{- include "rustfs.probes" $ | nindent 10 }} volumeMounts: - {{- if $.Values.mtls.enabled }} - - name: client-cert - mountPath: /opt/tls/client_cert.pem - subPath: client_cert.pem - - name: client-cert - mountPath: /opt/tls/client_key.pem - subPath: client_key.pem - - name: client-cert - mountPath: /opt/tls/client_ca.crt - subPath: client_ca.crt - - name: server-cert - mountPath: /opt/tls/rustfs_cert.pem - subPath: rustfs_cert.pem - - name: server-cert - mountPath: /opt/tls/rustfs_key.pem - subPath: rustfs_key.pem - - name: server-cert - mountPath: /opt/tls/ca.crt - subPath: ca.crt + {{- if or $.Values.mtls.enabled $tlsEnabled }} + - name: tls-material + mountPath: /opt/tls + readOnly: true {{- end }} {{- with $.Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -254,29 +241,42 @@ spec: - name: data mountPath: /data {{- end }} - {{- if or $.Values.mtls.enabled $.Values.extraVolumes }} + {{- if or $.Values.mtls.enabled $tlsEnabled $.Values.extraVolumes }} volumes: - {{- if $.Values.mtls.enabled }} - - name: server-cert - secret: - secretName: {{ include "rustfs.fullname" $ }}-server-tls - items: - - key: tls.crt - path: rustfs_cert.pem - - key: tls.key - path: rustfs_key.pem - - key: ca.crt - path: ca.crt - - name: client-cert - secret: - secretName: {{ include "rustfs.fullname" $ }}-client-tls - items: - - key: tls.crt - path: client_cert.pem - - key: tls.key - path: client_key.pem - - key: ca.crt - path: client_ca.crt + {{- if or $.Values.mtls.enabled $tlsEnabled }} + - name: tls-material + projected: + defaultMode: 0440 + sources: + - secret: + name: {{ if $.Values.mtls.enabled }}{{ include "rustfs.fullname" $ }}-server-tls{{ else }}{{ required "tls.existingSecret is required when tls.enabled=true" $tls.existingSecret }}{{ end }} + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem + {{- if $.Values.mtls.enabled }} + - key: ca.crt + path: ca.crt + {{- end }} + {{- if $.Values.mtls.enabled }} + - secret: + name: {{ include "rustfs.fullname" $ }}-client-tls + items: + - key: tls.crt + path: client_cert.pem + - key: tls.key + path: client_key.pem + - key: ca.crt + path: client_ca.crt + {{- end }} + {{- if and $tlsEnabled $tls.trustSecret }} + - secret: + name: {{ $tls.trustSecret }} + items: + - key: ca.crt + path: public.crt + {{- end }} {{- end }} {{- with $.Values.extraVolumes }} {{- toYaml . | nindent 8 }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 88d01e341..de0c4b7f1 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -388,6 +388,14 @@ gatewayApi: name: "" namespace: "" +# Native TLS identity managed by an external certificate controller. +# Public TLS and an mTLS backend use separate listeners or an ingress terminator. +tls: + enabled: false + existingSecret: "" # kubernetes.io/tls: tls.crt and tls.key only + trustSecret: "" # Optional independent outbound trust bundle: ca.crt + reloadInterval: 30 + mtls: enabled: false clientCertPath: "/opt/tls/client_cert.pem" diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 1f3f92469..fd9afef19 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -70,6 +70,7 @@ pub mod system; pub mod table_catalog; mod target_descriptor; pub mod tier; +pub mod tls_certificate; pub mod tls_debug; pub mod trace; pub mod usage_prefix; diff --git a/rustfs/src/admin/handlers/tls_certificate.rs b/rustfs/src/admin/handlers/tls_certificate.rs new file mode 100644 index 000000000..a45762625 --- /dev/null +++ b/rustfs/src/admin/handlers/tls_certificate.rs @@ -0,0 +1,177 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::{ + auth::authorize_admin_request, + router::{AdminOperation, Operation, S3Router}, + storage_api::s3::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}, +}; +use crate::server::{ + ADMIN_PREFIX, NativeTlsTransport, + tls_publish::{self, PublishError, PublishRequest}, +}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use rustfs_policy::policy::{action::Action, action::AdminAction}; + +pub fn register_tls_certificate_routes(router: &mut S3Router) -> std::io::Result<()> { + router.insert( + Method::GET, + &format!("{ADMIN_PREFIX}/v3/tls/certificate-state"), + AdminOperation(&CertificateStateHandler), + )?; + router.insert( + Method::POST, + &format!("{ADMIN_PREFIX}/v3/tls/certificate"), + AdminOperation(&PublishCertificateHandler), + )?; + Ok(()) +} + +pub struct CertificateStateHandler; +pub struct PublishCertificateHandler; + +fn require_native_tls(req: &S3Request) -> S3Result<()> { + if !req + .extensions + .get::() + .is_some_and(|transport| transport.0) + { + return Err(S3Error::with_message( + S3ErrorCode::AccessDenied, + "certificate management requires TLS on the RustFS listener", + )); + } + Ok(()) +} + +async fn authorize_certificate_request(req: &S3Request, action: AdminAction) -> S3Result<()> { + if req.credentials.is_none() { + return Err(S3Error::with_message( + S3ErrorCode::AccessDenied, + "certificate management requires credentials", + )); + } + authorize_admin_request(req, vec![Action::AdminAction(action)]).await?; + require_native_tls(req) +} + +fn tls_path() -> S3Result { + crate::server::tls_material::active_server_tls_path() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "certificate publication is not configured")) +} + +fn publication_error(error: PublishError) -> S3Error { + let (code, status) = match error { + PublishError::Conflict => (S3ErrorCode::InvalidRequest, StatusCode::CONFLICT), + PublishError::Disabled => (S3ErrorCode::InvalidRequest, StatusCode::SERVICE_UNAVAILABLE), + PublishError::Invalid => (S3ErrorCode::InvalidRequest, StatusCode::BAD_REQUEST), + PublishError::Storage | PublishError::Busy => (S3ErrorCode::InternalError, StatusCode::SERVICE_UNAVAILABLE), + }; + let mut result = S3Error::with_message(code, error.to_string()); + result.set_status_code(status); + result +} + +fn json_response(value: &impl serde::Serialize) -> S3Result> { + let bytes = serde_json::to_vec(value) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "certificate status serialization failed"))?; + let mut headers = HeaderMap::new(); + headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers)) +} + +#[async_trait::async_trait] +impl Operation for CertificateStateHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_certificate_request(&req, AdminAction::TLSCertificateStatusAdminAction).await?; + json_response(&tls_publish::certificate_state(tls_path()?).await.map_err(publication_error)?) + } +} + +#[async_trait::async_trait] +impl Operation for PublishCertificateHandler { + async fn call(&self, mut req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_certificate_request(&req, AdminAction::TLSCertificatePublishAdminAction).await?; + let body = req.input.store_all_limited(128 * 1024).await.map_err(|_| { + S3Error::with_message( + S3ErrorCode::InvalidRequest, + "certificate publication body exceeds the limit or is unreadable", + ) + })?; + let request: PublishRequest = serde_json::from_slice(&body) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "invalid certificate publication JSON"))?; + json_response(&tls_publish::publish(tls_path()?, request).await.map_err(publication_error)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(method: Method) -> S3Request { + S3Request { + input: Body::empty(), + method, + uri: "/rustfs/admin/v3/tls/certificate".parse().expect("URI"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } + } + + #[tokio::test] + async fn certificate_handlers_reject_missing_credentials() { + assert_eq!( + CertificateStateHandler + .call(request(Method::GET), Params::new()) + .await + .expect_err("unauthenticated GET") + .code(), + &S3ErrorCode::AccessDenied + ); + assert_eq!( + PublishCertificateHandler + .call(request(Method::POST), Params::new()) + .await + .expect_err("unauthenticated POST") + .code(), + &S3ErrorCode::AccessDenied + ); + } + + #[test] + fn forwarded_protocol_never_authorizes_private_key_upload() { + let mut request = request(Method::POST); + request.headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + assert!(require_native_tls(&request).is_err()); + request.extensions.insert(NativeTlsTransport(false)); + assert!(require_native_tls(&request).is_err()); + request.extensions.insert(NativeTlsTransport(true)); + assert!(require_native_tls(&request).is_ok()); + } + + #[test] + fn unknown_json_fields_are_rejected_without_echoing_material() { + let error = serde_json::from_str::( + r#"{"schema_version":"x","resource_id":"x","expected_target_revision":"x","issuance_id":"00000000-0000-0000-0000-000000000001","certificate_pem":"cert","private_key_pem":"secret","extra":true}"#, + ); + assert!(error.is_err()); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d330221d6..b9fb0373d 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -103,6 +103,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() inspect_archive::register_inspect_archive_route(r)?; gateway_key_inventory::register_gateway_key_inventory_route(r)?; tls_debug::register_tls_debug_route(r)?; + handlers::tls_certificate::register_tls_certificate_routes(r)?; kms::register_kms_route(r)?; oidc::register_oidc_route(r)?; idp_compat::register_idp_compat_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index d3baf28d2..8acd5f313 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -21,6 +21,8 @@ const ALL_ADMIN: AdminActionRef = AdminActionRef::new("AllAdminActions"); const ADD_USER_TO_GROUP: AdminActionRef = AdminActionRef::new("AddUserToGroupAdminAction"); const ATTACH_POLICY: AdminActionRef = AdminActionRef::new("AttachPolicyAdminAction"); const CONFIG_UPDATE: AdminActionRef = AdminActionRef::new("ConfigUpdateAdminAction"); +const TLS_CERTIFICATE_PUBLISH: AdminActionRef = AdminActionRef::new("TLSCertificatePublishAdminAction"); +const TLS_CERTIFICATE_STATUS: AdminActionRef = AdminActionRef::new("TLSCertificateStatusAdminAction"); const CONSOLE_LOG: AdminActionRef = AdminActionRef::new("ConsoleLogAdminAction"); const COMMIT_TABLE: AdminActionRef = AdminActionRef::new("CommitTableAction"); const CREATE_POLICY: AdminActionRef = AdminActionRef::new("CreatePolicyAdminAction"); @@ -501,6 +503,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ ), admin(HttpMethod::Get, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), admin(HttpMethod::Put, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/tls/certificate-state", + TLS_CERTIFICATE_STATUS, + RouteRiskLevel::Sensitive, + ), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/tls/certificate", + TLS_CERTIFICATE_PUBLISH, + RouteRiskLevel::High, + ), admin(HttpMethod::Get, "/rustfs/admin/v3/scanner/status", SERVER_INFO, RouteRiskLevel::Sensitive), admin( HttpMethod::Post, @@ -2233,6 +2247,14 @@ mod tests { } } + #[test] + fn route_policy_records_dedicated_tls_actions() { + assert_action(HttpMethod::Get, "/rustfs/admin/v3/tls/certificate-state", TLS_CERTIFICATE_STATUS); + assert_action(HttpMethod::Post, "/rustfs/admin/v3/tls/certificate", TLS_CERTIFICATE_PUBLISH); + assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/tls/certificate", CONFIG_UPDATE); + assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/tls/certificate-state", TLS_CERTIFICATE_PUBLISH); + } + /// `Critical` marks the routes whose worst case is permanent loss of user /// data, so the inventory is pinned in both directions: the KMS key /// deletion route may not be quietly downgraded, and no other route may be diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 3d85e3c11..ac24a3d6f 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -302,6 +302,8 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::PUT, "/v3/restore-config-history-kv"), admin_route(Method::GET, "/v3/config"), admin_route(Method::PUT, "/v3/config"), + admin_route(Method::GET, "/v3/tls/certificate-state"), + admin_route(Method::POST, "/v3/tls/certificate"), admin_route(Method::GET, "/v3/scanner/status"), admin_route(Method::POST, "/v3/scanner/cycle-state/reset"), admin_route(Method::POST, "/v3/scanner/usage-state/reset"), diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 8feed740a..c93f7bdd1 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -32,12 +32,12 @@ use crate::server::{ ssec_transport::SsecTransportLayer, strip_valid_port_suffix, tls_material::{ - TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, build_acceptor_from_loaded, + PreparedServerTls, TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, load_tls_material, spawn_reload_loop, }, }; #[cfg(feature = "http3")] -use crate::server::{http3, hybrid::HybridBody, tls_material::build_http3_server_config_from_loaded}; +use crate::server::{http3, hybrid::HybridBody}; use crate::storage_api::server::http as storage; use crate::storage_api::server::http::rpc::InternodeRpcService; #[cfg(test)] @@ -1448,6 +1448,7 @@ pub async fn start_http_server( // Load TLS materials and build server acceptor in a single pass. // Outbound material (root CAs, mTLS identity) was already published in main.rs; // this load is needed for the server-side TLS acceptor and reload loop. + let mut tls_initial_snapshot = None; let tls_acceptor = if tls_path_configured { let snapshot = load_tls_material(tls_path).await.map_err(|e| { Error::other(format!( @@ -1456,27 +1457,30 @@ pub async fn start_http_server( )) })?; - #[cfg(feature = "http3")] - if http3_enabled { - http3_server_config = build_http3_server_config_from_loaded(snapshot.server.as_ref(), std::path::Path::new(tls_path)) - .await - .map_err(|e| Error::other(format!("HTTP/3 TLS configuration failed: {e}")))?; - } - - let acceptor = build_acceptor_from_loaded(snapshot.server, std::path::Path::new(tls_path)) - .await - .map_err(|e| Error::other(e.to_string()))?; + let prepared = PreparedServerTls::from_snapshot(&snapshot).map_err(|e| Error::other(e.to_string()))?; // Fail closed: if TLS was explicitly configured but no server certificates // were found, refuse to start rather than silently falling back to plain HTTP. - match acceptor { + match prepared { None => { return Err(Error::other(format!( "TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but no server certificates were found", tls_path ))); } - Some(a) => Some(a), + Some(prepared) => { + #[cfg(feature = "http3")] + if http3_enabled { + http3_server_config = Some( + prepared + .http3_config() + .map_err(|e| Error::other(format!("HTTP/3 TLS configuration failed: {e}")))?, + ); + } + let acceptor = prepared.tcp_acceptor().map_err(|e| Error::other(e.to_string()))?; + tls_initial_snapshot = Some((snapshot, prepared)); + Some(acceptor) + } } } else { None @@ -1755,14 +1759,20 @@ pub async fn start_http_server( } else { None }; - let tls_reload_task = tls_acceptor.as_ref().and_then(|holder| { + let tls_reload_task = if let (Some(holder), Some((initial, prepared))) = (tls_acceptor.as_ref(), tls_initial_snapshot.take()) + { spawn_reload_loop( tls_path.to_string(), holder.clone(), + initial, + prepared, #[cfg(feature = "http3")] http3_server.as_ref().map(|(endpoint, _)| endpoint.clone()), ) - }); + .map_err(|e| Error::other(e.to_string()))? + } else { + None + }; let is_console = config.console_enable; let server_domains_configured = !config.server_domains.is_empty(); diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 7f919d6f7..4841bf4fe 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -29,6 +29,7 @@ pub(crate) mod runtime_sources; mod service_state; mod ssec_transport; pub mod tls_material; +pub(crate) mod tls_publish; #[cfg(feature = "http3")] mod http3; @@ -85,6 +86,7 @@ pub(crate) use readiness::collect_node_readiness_report; pub use readiness::publish_ready_when_runtime_ready; pub(crate) use readiness::snapshot_dependency_readiness_report; pub(crate) use readiness::{collect_cluster_read_health_report, collect_cluster_write_health_report}; +pub(crate) use ssec_transport::NativeTlsTransport; pub use crate::shared_types::RemoteAddr; diff --git a/rustfs/src/server/ssec_transport.rs b/rustfs/src/server/ssec_transport.rs index bcda8f08a..bbe7adaba 100644 --- a/rustfs/src/server/ssec_transport.rs +++ b/rustfs/src/server/ssec_transport.rs @@ -63,6 +63,10 @@ pub(crate) const METRIC_SSEC_PLAINTEXT_REQUESTS_TOTAL: &str = "rustfs_ssec_plain type BoxError = Box; type BoxBody = http_body_util::combinators::UnsyncBoxBody; +/// Set only by the listener stack; proxy protocol assertions do not establish this marker. +#[derive(Clone, Copy)] +pub(crate) struct NativeTlsTransport(pub(crate) bool); + /// Every header that carries, or announces, a customer-provided key: the /// object's own triple and the copy-source triple that CopyObject and /// UploadPartCopy use to read an SSE-C source. @@ -191,7 +195,8 @@ where self.inner.poll_ready(cx) } - fn call(&mut self, req: Request) -> Self::Future { + fn call(&mut self, mut req: Request) -> Self::Future { + req.extensions_mut().insert(NativeTlsTransport(self.connection_is_tls)); if !carries_ssec_headers(req.headers()) || is_secure_transport(self.connection_is_tls, req.extensions().get::()) { diff --git a/rustfs/src/server/tls_material.rs b/rustfs/src/server/tls_material.rs index 4d6239eb2..1790ee2a9 100644 --- a/rustfs/src/server/tls_material.rs +++ b/rustfs/src/server/tls_material.rs @@ -26,19 +26,20 @@ use rustfs_config::{ DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_RUSTFS_EXTRA_CA_CERT, ENV_SERVER_MTLS_ENABLE, ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, - ENV_TRUST_SYSTEM_CA, RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, - RUSTFS_TLS_CERT, -}; -use rustfs_tls_runtime::{ - ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions, - build_webpki_client_verifier, create_multi_cert_resolver, + ENV_TRUST_SYSTEM_CA, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, }; +#[cfg(test)] +use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME}; +use rustfs_tls_runtime::{ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, create_multi_cert_resolver}; +#[cfg(test)] +use rustfs_tls_runtime::{WebPkiClientVerifierOptions, build_webpki_client_verifier}; use rustfs_utils::{get_env_bool, get_env_opt_str}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Cursor; use std::path::{Path, PathBuf}; -use std::sync::Arc; use std::sync::RwLock; +use std::sync::{Arc, LazyLock, Mutex, Weak}; use std::time::Duration; use tokio_rustls::TlsAcceptor; use tracing::{debug, info, warn}; @@ -75,11 +76,20 @@ pub async fn load_tls_material(tls_path: &str) -> Result Result, tls_dir: &Path, @@ -100,76 +111,166 @@ pub(crate) async fn build_acceptor_from_loaded( ) .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - match server { - Some(RuntimeServerTlsMaterial::SingleCert { certs, key }) => { - let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier, ServerProtocol::Tcp)?; - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_acceptor_created", - mode = "single_cert", - "TLS acceptor created" - ); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) + build_acceptor_with_verifier(server, mtls_verifier) +} + +fn snapshot_client_verifier( + snapshot: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result>, TlsMaterialError> { + if !get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE) { + return Ok(None); + } + let pem = snapshot + .server_client_ca_pem + .as_deref() + .ok_or_else(|| TlsMaterialError::Parse("mTLS client trust is missing".into()))?; + let mut reader = Cursor::new(pem); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_reader_iter(&mut reader) { + roots + .add(cert.map_err(|e| TlsMaterialError::Parse(e.to_string()))?) + .map_err(|e| TlsMaterialError::Parse(e.to_string()))?; + } + let verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(roots)) + .build() + .map_err(|e| TlsMaterialError::Parse(e.to_string()))?; + #[cfg(test)] + TLS_PREPARE_COUNTS.with(|counts| { + let (roots, keys) = counts.get(); + counts.set((roots + 1, keys)); + }); + Ok(Some(verifier)) +} + +#[cfg(test)] +thread_local! { + static TLS_PREPARE_COUNTS: std::cell::Cell<(usize, usize)> = const { std::cell::Cell::new((0, 0)) }; +} + +/// Expensive certificate signing keys and client trust are prepared once per generation. +/// TCP and QUIC retain this shared resolver while owning their protocol settings. +#[derive(Clone)] +pub(crate) struct PreparedServerTls { + resolver: Arc, + verifier: Option>, + provider: Arc, +} + +impl PreparedServerTls { + pub(crate) fn from_snapshot(snapshot: &rustfs_tls_runtime::TlsMaterialSnapshot) -> Result, TlsMaterialError> { + Self::from_material(snapshot.server.as_ref(), snapshot_client_verifier(snapshot)?) + } + + fn from_material( + server: Option<&RuntimeServerTlsMaterial>, + verifier: Option>, + ) -> Result, TlsMaterialError> { + let Some(server) = server else { + return Ok(None); + }; + let provider = rustls::ServerConfig::builder().crypto_provider().clone(); + let (resolver, validity): (Arc, _) = match server { + RuntimeServerTlsMaterial::SingleCert { certs, key } => { + let validity = certificate_validities(std::iter::once(certs.as_slice()))?; + let certified = rustls::sign::CertifiedKey::from_der(certs.clone(), key.clone_key(), &provider) + .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?; + (Arc::new(rustls::sign::SingleCertAndKey::from(certified)), validity) + } + RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } => { + let validity = certificate_validities(cert_key_pairs.values().map(|(certs, _)| certs.as_slice()))?; + let pairs = cert_key_pairs + .iter() + .map(|(name, (certs, key))| (name.clone(), (certs.clone(), key.clone_key()))) + .collect(); + let resolver = create_multi_cert_resolver(pairs) + .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; + (Arc::new(resolver), validity) + } + }; + #[cfg(test)] + TLS_PREPARE_COUNTS.with(|counts| { + let (roots, keys) = counts.get(); + counts.set((roots, keys + 1)); + }); + Ok(Some(Self { + resolver: Arc::new(ValidityCheckedResolver { + inner: resolver, + validity, + }), + verifier, + provider, + })) + } + + pub(crate) fn tcp_acceptor(&self) -> Result, TlsMaterialError> { + let config = self.server_config(ServerProtocol::Tcp)?; + Ok(Arc::new(TlsAcceptorHolder::new(Arc::new(TlsAcceptor::from(Arc::new(config)))))) + } + + #[cfg(feature = "http3")] + pub(crate) fn http3_config(&self) -> Result { + let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(self.server_config(ServerProtocol::Http3)?) + .map_err(|e| TlsMaterialError::Io(format!("configure QUIC TLS: {e}")))?; + Ok(quinn::ServerConfig::with_crypto(Arc::new(crypto))) + } + + fn server_config(&self, protocol: ServerProtocol) -> Result { + let versions = match protocol { + ServerProtocol::Tcp => rustls::DEFAULT_VERSIONS, + #[cfg(feature = "http3")] + ServerProtocol::Http3 => &[&rustls::version::TLS13], + }; + let builder = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(versions) + .map_err(|e| TlsMaterialError::Io(format!("configure TLS protocol: {e}")))?; + let mut config = if let Some(verifier) = &self.verifier { + builder + .with_client_cert_verifier(verifier.clone()) + .with_cert_resolver(self.resolver.clone()) + } else { + builder.with_no_client_auth().with_cert_resolver(self.resolver.clone()) + }; + config.alpn_protocols = match protocol { + ServerProtocol::Tcp => vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()], + #[cfg(feature = "http3")] + ServerProtocol::Http3 => vec![b"h3".to_vec()], + }; + #[cfg(feature = "http3")] + if matches!(protocol, ServerProtocol::Http3) { + config.max_early_data_size = 0; } - Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { - let resolver = create_multi_cert_resolver(cert_key_pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; - let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier, ServerProtocol::Tcp)?; - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_acceptor_created", - mode = "sni_resolver", - "TLS acceptor created" - ); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) + config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); + if tls_key_log() { + config.key_log = Arc::new(rustls::KeyLogFile::new()); } - None => Ok(None), + Ok(config) } } -#[cfg(feature = "http3")] +#[cfg(test)] +fn build_acceptor_with_verifier( + server: Option, + mtls_verifier: Option>, +) -> Result>, TlsMaterialError> { + PreparedServerTls::from_material(server.as_ref(), mtls_verifier)? + .map(|prepared| prepared.tcp_acceptor()) + .transpose() +} + +#[cfg(all(test, feature = "http3"))] pub(crate) async fn build_http3_server_config_from_loaded( server: Option<&RuntimeServerTlsMaterial>, tls_dir: &Path, ) -> Result, TlsMaterialError> { - let Some(server) = server else { - return Ok(None); - }; - let mtls_verifier = build_webpki_client_verifier( WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) .build(), ) .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - - let source = match server { - RuntimeServerTlsMaterial::SingleCert { certs, key } => ServerCertSource::SingleCert { - certs: certs.clone(), - key: key.clone_key(), - }, - RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } => { - let pairs = cert_key_pairs - .iter() - .map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key()))) - .collect(); - - let resolver = create_multi_cert_resolver(pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build HTTP/3 SNI resolver: {e}")))?; - - ServerCertSource::Resolver(Arc::new(resolver)) - } - }; - - let tls = build_server_config(source, mtls_verifier, ServerProtocol::Http3)?; - let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(tls) - .map_err(|e| TlsMaterialError::Io(format!("configure QUIC TLS: {e}")))?; - - Ok(Some(quinn::ServerConfig::with_crypto(Arc::new(crypto)))) + PreparedServerTls::from_material(server, mtls_verifier)? + .map(|prepared| prepared.http3_config()) + .transpose() } // ── Outbound Enrichment ── @@ -179,18 +280,30 @@ pub(crate) async fn build_http3_server_config_from_loaded( /// 1. Optional: server leaf cert as root CA (`RUSTFS_TRUST_LEAF_CERT_AS_CA`) /// 2. Optional: system root CAs from platform paths (`RUSTFS_TRUST_SYSTEM_CA`) /// 3. Optional: mTLS identity from env-var-overridden paths -async fn enrich_outbound(outbound: &mut rustfs_tls_runtime::OutboundTlsMaterial, tls_dir: &Path) -> Result<(), TlsMaterialError> { - // 1. Optional: load leaf certs as root CAs - if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) - && load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut outbound.root_ca_pem).await - { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_root_ca_enriched", - source = "leaf_certificate", - "TLS root CA set enriched" - ); +async fn enrich_outbound(snapshot: &mut rustfs_tls_runtime::TlsMaterialSnapshot) -> Result<(), TlsMaterialError> { + let tls_dir = &snapshot.source.base_dir; + let outbound = &mut snapshot.outbound; + if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) { + let chains: Vec<_> = match &snapshot.server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, .. }) => vec![certs], + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { + cert_key_pairs.values().map(|(certs, _)| certs).collect() + } + None => Vec::new(), + }; + let mut certificates = BTreeSet::new(); + for chain in chains { + certificates.extend(chain.iter().map(|cert| cert.as_ref().to_vec())); + } + for certificate in certificates { + outbound.root_ca_pem.extend_from_slice( + format!( + "-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n", + base64_simd::STANDARD.encode_to_string(&certificate), + ) + .as_bytes(), + ); + } } // 2. Optional: load system root CAs @@ -260,19 +373,6 @@ async fn load_mtls_identity_from_overridden_paths( None => tls_dir.join(RUSTFS_CLIENT_KEY_FILENAME), }; - if !client_cert_path.exists() || !client_key_path.exists() { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "mtls_identity_state", - state = "not_configured", - cert_path = ?client_cert_path, - key_path = ?client_key_path, - "mTLS client identity not configured" - ); - return Ok(None); - } - let cert_pem = tokio::fs::read(&client_cert_path) .await .map_err(|e| TlsMaterialError::Io(format!("read client cert {client_cert_path:?}: {e}")))?; @@ -281,11 +381,13 @@ async fn load_mtls_identity_from_overridden_paths( .map_err(|e| TlsMaterialError::Io(format!("read client key {client_key_path:?}: {e}")))?; let mut reader = std::io::Cursor::new(&cert_pem); - if CertificateDer::pem_reader_iter(&mut reader).next().is_none() { - return Err(TlsMaterialError::Parse("no valid certificate in client cert PEM".into())); - } + let certs = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| TlsMaterialError::Parse(format!("invalid client certificate: {e}")))?; let mut reader = std::io::Cursor::new(&key_pem); - PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| TlsMaterialError::Parse(format!("invalid client key PEM: {e}")))?; + let key = + PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| TlsMaterialError::Parse(format!("invalid client key: {e}")))?; + rustfs_tls_runtime::validate_certified_key(&certs, &key, "client identity override", false).map_err(map_runtime_tls_error)?; info!( component = LOG_COMPONENT_TLS, @@ -413,54 +515,47 @@ async fn load_cert_file(path: &Path, pem_data: &mut Vec, desc: &str) -> bool /// Search for and load certificate files matching `cert_name` in the directory /// and one level of subdirectories. /// Returns `true` if at least one matching file was loaded. -async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec) -> bool { - let Ok(mut rd) = tokio::fs::read_dir(dir).await else { - debug!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_material_directory_state", - path = %dir.display(), - state = "missing", - "TLS material directory state changed" + +type CertificateValidities = HashMap, (i64, i64)>; + +fn certificate_validities<'a>( + chains: impl Iterator]>, +) -> Result { + let mut validity = HashMap::new(); + for chain in chains { + let leaf = chain + .first() + .ok_or_else(|| TlsMaterialError::Parse("empty certificate chain".into()))?; + let (_, parsed) = x509_parser::parse_x509_certificate(leaf.as_ref()) + .map_err(|e| TlsMaterialError::Parse(format!("invalid certificate: {e}")))?; + validity.insert( + leaf.as_ref().to_vec(), + (parsed.validity().not_before.timestamp(), parsed.validity().not_after.timestamp()), ); - return false; - }; - - let mut loaded = false; - while let Ok(Some(entry)) = rd.next_entry().await { - let Ok(ft) = entry.file_type().await else { continue }; - - if ft.is_file() { - let fname = entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&entry.path(), pem_data, "certificate").await { - loaded = true; - } - } else if ft.is_dir() - && let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await - { - while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { - if let Ok(sub_ft) = sub_entry.file_type().await - && sub_ft.is_file() - { - let fname = sub_entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await { - loaded = true; - } - } - } - } } - loaded + Ok(validity) } -// ── Server Config Construction ── +#[derive(Debug)] +struct ValidityCheckedResolver { + inner: Arc, + validity: CertificateValidities, +} -enum ServerCertSource { - Resolver(Arc), - SingleCert { - certs: Vec>, - key: PrivateKeyDer<'static>, - }, +impl rustls::server::ResolvesServerCert for ValidityCheckedResolver { + fn resolve(&self, hello: rustls::server::ClientHello<'_>) -> Option> { + let certificate = self.inner.resolve(hello)?; + let leaf = certificate.cert.first()?; + let (not_before, not_after) = self.validity.get(leaf.as_ref())?; + let now = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(), + ) + .ok()?; + (*not_before <= now && now < *not_after).then_some(certificate) + } } enum ServerProtocol { @@ -469,62 +564,6 @@ enum ServerProtocol { Http3, } -fn build_server_config( - cert_source: ServerCertSource, - mtls_verifier: Option>, - protocol: ServerProtocol, -) -> Result { - let builder = match protocol { - ServerProtocol::Tcp => rustls::ServerConfig::builder(), - #[cfg(feature = "http3")] - ServerProtocol::Http3 => rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]), - }; - - let mut config = match cert_source { - ServerCertSource::Resolver(resolver) => { - if let Some(verifier) = mtls_verifier { - builder.with_client_cert_verifier(verifier).with_cert_resolver(resolver) - } else { - builder.with_no_client_auth().with_cert_resolver(resolver) - } - } - ServerCertSource::SingleCert { certs, key } => { - if let Some(verifier) = mtls_verifier { - builder - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))? - } else { - builder - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))? - } - } - }; - - config.alpn_protocols = match protocol { - ServerProtocol::Tcp => { - vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()] - } - #[cfg(feature = "http3")] - ServerProtocol::Http3 => vec![b"h3".to_vec()], - }; - - #[cfg(feature = "http3")] - if matches!(protocol, ServerProtocol::Http3) { - config.max_early_data_size = 0; - } - - config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); - - if tls_key_log() { - config.key_log = Arc::new(rustls::KeyLogFile::new()); - } - - Ok(config) -} - fn tls_key_log() -> bool { get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG) } @@ -647,26 +686,30 @@ impl TlsAcceptorHolder { } // Build both configurations before publishing either listener's new certificate. +#[cfg(all(test, feature = "http3"))] async fn reload_acceptors( server: Option, tls_dir: &Path, holder: &TlsAcceptorHolder, #[cfg(feature = "http3")] http3_endpoint: Option<&quinn::Endpoint>, ) -> Result { - #[cfg(feature = "http3")] + let verifier = build_webpki_client_verifier( + WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) + .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) + .build(), + ) + .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; + let Some(prepared) = PreparedServerTls::from_material(server.as_ref(), verifier)? else { + return Ok(false); + }; let http3_config = if http3_endpoint.is_some() { - let config = build_http3_server_config_from_loaded(server.as_ref(), tls_dir).await?; - let Some(mut config) = config else { - return Ok(false); - }; + let mut config = prepared.http3_config()?; config.migration(false); Some(config) } else { None }; - let Some(new_holder) = build_acceptor_from_loaded(server, tls_dir).await? else { - return Ok(false); - }; + let new_holder = prepared.tcp_acceptor()?; #[cfg(feature = "http3")] if let (Some(endpoint), Some(config)) = (http3_endpoint, http3_config) { endpoint.set_server_config(Some(config)); @@ -680,146 +723,327 @@ async fn reload_acceptors( /// Spawn a background task that periodically checks for TLS certificate changes. /// Single load per tick: loads once via tls-runtime, enriches, publishes outbound, /// and builds acceptor — no double reads. +#[derive(Clone)] +struct ReloadTarget { + holder: Arc, + #[cfg(feature = "http3")] + http3_endpoint: Option, +} + +struct ReloadState { + targets: HashMap, + accepted: Arc, + tcp: Arc, + prepared: PreparedServerTls, + #[cfg(feature = "http3")] + quic: Option, +} + +struct ReloadGroup { + required_startup_files: BTreeSet, + state: Mutex, + changed: tokio::sync::Notify, + next_id: std::sync::atomic::AtomicU64, + #[cfg(test)] + pass_completed: tokio::sync::Notify, +} + +// Lock order is registry, then group state, then acceptor. Neither guard crosses an await. +static RELOAD_GROUPS: LazyLock>>> = LazyLock::new(|| Mutex::new(HashMap::new())); + +pub(crate) fn observed_server_snapshot(tls_path: &str) -> Option> { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + let group = registry.get(tls_path)?.upgrade()?; + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.targets.is_empty()).then(|| state.accepted.clone()) +} + +/// Resolve the source actually registered by listener configuration, including CLI overrides. +pub(crate) fn active_server_tls_path() -> Option { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + let mut paths = registry.iter().filter_map(|(path, group)| { + let group = group.upgrade()?; + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.targets.is_empty()).then(|| path.clone()) + }); + let path = paths.next()?; + paths.next().is_none().then_some(path) +} + +pub(crate) fn notify_server_publication(tls_path: &str) { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(group) = registry.get(tls_path).and_then(Weak::upgrade) { + group.changed.notify_one(); + } +} + +struct ReloadRegistration { + group: Arc, + id: u64, +} + +impl Drop for ReloadRegistration { + fn drop(&mut self) { + self.group + .state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .targets + .remove(&self.id); + self.group.changed.notify_one(); + } +} + +fn server_names(server: &Option) -> Result>, TlsMaterialError> { + let chains: Vec<_> = match server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, .. }) => vec![("default", certs)], + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => cert_key_pairs + .iter() + .map(|(selector, (certs, _))| (selector.as_str(), certs)) + .collect(), + None => Vec::new(), + }; + let mut selectors = BTreeMap::new(); + for (selector, chain) in chains { + let leaf = chain + .first() + .ok_or_else(|| TlsMaterialError::Parse("empty certificate chain".into()))?; + selectors.insert(selector.to_string(), certificate_names(leaf)?); + } + Ok(selectors) +} + +pub(crate) fn certificate_names(leaf: &CertificateDer<'_>) -> Result, TlsMaterialError> { + let mut names = BTreeSet::new(); + let (_, parsed) = x509_parser::parse_x509_certificate(leaf.as_ref()) + .map_err(|e| TlsMaterialError::Parse(format!("invalid certificate: {e}")))?; + if let Some(san) = parsed + .subject_alternative_name() + .map_err(|e| TlsMaterialError::Parse(format!("invalid SAN: {e}")))? + { + for name in &san.value.general_names { + match name { + x509_parser::extensions::GeneralName::DNSName(name) => { + names.insert(format!("dns:{}", name.to_ascii_lowercase())); + } + x509_parser::extensions::GeneralName::IPAddress(bytes) => { + names.insert(format!("ip:{bytes:?}")); + } + _ => {} + } + } + } + Ok(names) +} + +pub(crate) fn validate_required_material_files( + required: &BTreeSet, + candidate: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result<(), TlsMaterialError> { + if !required.is_subset(&candidate.material_files) { + return Err(TlsMaterialError::Parse("required TLS source components disappeared".into())); + } + Ok(()) +} + +pub(crate) fn validate_candidate_replacement( + accepted: &rustfs_tls_runtime::TlsMaterialSnapshot, + candidate: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result<(), TlsMaterialError> { + validate_required_material_files(&accepted.material_files, candidate)?; + let old_names = server_names(&accepted.server)?; + let new_names = server_names(&candidate.server)?; + for (selector, names) in old_names { + if !new_names.get(&selector).is_some_and(|new| names.is_subset(new)) { + return Err(TlsMaterialError::Parse("required server certificate names disappeared".into())); + } + } + Ok(()) +} + +fn outbound_material_matches(a: &rustfs_tls_runtime::OutboundTlsMaterial, b: &rustfs_tls_runtime::OutboundTlsMaterial) -> bool { + a.root_ca_pem == b.root_ca_pem && a.mtls_identity == b.mtls_identity +} + pub(crate) fn spawn_reload_loop( tls_path: String, holder: Arc, + initial: rustfs_tls_runtime::TlsMaterialSnapshot, + prepared: PreparedServerTls, #[cfg(feature = "http3")] http3_endpoint: Option, -) -> Option> { - let enabled = get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE); - if !enabled { - debug!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "disabled", - env_var = ENV_TLS_RELOAD_ENABLE, - "TLS reload state changed" - ); - return None; +) -> Result>, TlsMaterialError> { + if !get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE) { + return Ok(None); } + let required_startup_files = crate::startup_tls_material::required_startup_material_files(&tls_path); + let initial = Arc::new(initial); + let mut registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + registry.retain(|_, group| group.strong_count() != 0); + let existing = registry.get(&tls_path).and_then(Weak::upgrade); + let first = existing.is_none(); + let group = existing.unwrap_or_else(|| { + Arc::new(ReloadGroup { + required_startup_files, + state: Mutex::new(ReloadState { + targets: HashMap::new(), + accepted: initial.clone(), + tcp: holder.get(), + prepared, + #[cfg(feature = "http3")] + quic: None, + }), + changed: tokio::sync::Notify::new(), + next_id: std::sync::atomic::AtomicU64::new(1), + #[cfg(test)] + pass_completed: tokio::sync::Notify::new(), + }) + }); + let id = group.next_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + { + let mut state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + #[cfg(feature = "http3")] + if let Some(endpoint) = &http3_endpoint { + if state.quic.is_none() { + let mut config = state.prepared.http3_config()?; + config.migration(false); + state.quic = Some(config); + } + endpoint.set_server_config(state.quic.clone()); + } + holder.swap(&TlsAcceptorHolder::new(state.tcp.clone())); + state.targets.insert( + id, + ReloadTarget { + holder, + #[cfg(feature = "http3")] + http3_endpoint, + }, + ); + } + registry.insert(tls_path.clone(), Arc::downgrade(&group)); + drop(registry); + if first { + tokio::spawn(run_reload_group(tls_path, group.clone())); + } + group.changed.notify_one(); + let registration = ReloadRegistration { group, id }; + Ok(Some(tokio::spawn(async move { + let _registration = registration; + std::future::pending::<()>().await; + }))) +} +async fn run_reload_group(tls_path: String, group: Arc) { let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5); - let tls_source = TlsSource::from_directory(&tls_path); - - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "enabled", - interval_secs, - "TLS reload state changed" - ); - - Some(tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); - let tls_dir = PathBuf::from(&tls_path); - loop { - interval.tick().await; - - match rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source).await { - Ok(mut snapshot) => { - if let Err(e) = enrich_outbound(&mut snapshot.outbound, &tls_dir).await { - startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "enrich_err", None, None); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "enrich_outbound", - error = %e, - "TLS reload failed" - ); - continue; - } - - let generation = startup_runtime_sources::current_outbound_tls_generation().saturating_add(1); - startup_runtime_sources::publish_outbound_tls_state(TlsGeneration(generation), &snapshot.outbound).await; - startup_runtime_sources::record_tls_generation("rustfs_server_reload_loop", generation); - if !snapshot.outbound.root_ca_pem.is_empty() { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_root_ca_enriched", - source = "reload_snapshot", - generation, - "TLS root CA set updated" - ); - } - - match reload_acceptors( - snapshot.server, - &tls_dir, - &holder, - #[cfg(feature = "http3")] - http3_endpoint.as_ref(), - ) - .await - { - Ok(true) => { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "reloaded", - generation, - "TLS reload state changed" - ); - startup_runtime_sources::record_tls_reload_result( - "rustfs_server_reload_loop", - "ok", - None, - Some(generation), - ); - } - Ok(false) => { - startup_runtime_sources::record_tls_reload_skipped("rustfs_server_reload_loop", "no_acceptor"); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "build_acceptor", - reason = "no_acceptor", - "TLS reload returned no acceptor" - ) - } - Err(e) => { - startup_runtime_sources::record_tls_reload_result( - "rustfs_server_reload_loop", - "acceptor_err", - None, - Some(generation), - ); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "build_acceptor", - error = %e, - generation, - "TLS reload failed" - ) - } + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut generation = 1_u64; + loop { + tokio::select! { + _ = interval.tick() => {}, + _ = group.changed.notified() => {}, + } + let (accepted, empty) = { + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (state.accepted.clone(), state.targets.is_empty()) + }; + if empty { + let mut registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + if group.state.lock().unwrap_or_else(|e| e.into_inner()).targets.is_empty() { + registry.remove(&tls_path); + return; + } + continue; + } + let result = async { + let candidate = load_tls_material(&tls_path).await?; + validate_candidate_replacement(&accepted, &candidate)?; + validate_required_material_files(&group.required_startup_files, &candidate)?; + // Listener initialization can load a newer bundle than startup published. + let published = startup_runtime_sources::current_outbound_tls_state().await; + let required_roots = + !accepted.outbound.root_ca_pem.is_empty() || published.root_ca_pem.as_ref().is_some_and(|pem| !pem.is_empty()); + let required_identity = accepted.outbound.mtls_identity.is_some() || published.mtls_identity.is_some(); + if (required_roots && candidate.outbound.root_ca_pem.is_empty()) + || (required_identity && candidate.outbound.mtls_identity.is_none()) + { + return Err(TlsMaterialError::Parse("required outbound TLS material disappeared".into())); + } + let outbound_changed = !outbound_material_matches(&accepted.outbound, &candidate.outbound) + || published.root_ca_pem.as_deref().unwrap_or_default() != candidate.outbound.root_ca_pem.as_slice() + || published.mtls_identity != candidate.outbound.mtls_identity; + if accepted.fingerprint == candidate.fingerprint && !outbound_changed { + startup_runtime_sources::record_tls_reload_skipped("rustfs_server_reload_loop", "unchanged"); + return Ok(None); + } + let prepared = PreparedServerTls::from_snapshot(&candidate)? + .ok_or_else(|| TlsMaterialError::Parse("server certificate is missing".into()))?; + let tcp = prepared.tcp_acceptor()?; + #[cfg(feature = "http3")] + let quic = { + let mut config = prepared.http3_config()?; + config.migration(false); + config + }; + let candidate = Arc::new(candidate); + let listeners = { + let mut state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + // Newly registered listeners join the same prepared generation under this lock. + for target in state.targets.values() { + #[cfg(feature = "http3")] + if let Some(endpoint) = &target.http3_endpoint { + endpoint.set_server_config(Some(quic.clone())); } + target.holder.swap(&tcp); } - Err(e) => { - startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "load_err", None, None); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "load_snapshot", - error = %e, - "TLS reload failed" - ); + state.tcp = tcp.get(); + state.prepared = prepared; + state.accepted = candidate.clone(); + #[cfg(feature = "http3")] + { + state.quic = Some(quic); } + state.targets.len() + }; + if outbound_changed { + let next = startup_runtime_sources::current_outbound_tls_generation().saturating_add(1); + startup_runtime_sources::publish_outbound_tls_state(TlsGeneration(next), &candidate.outbound).await; + } + Ok(Some(listeners)) + } + .await; + match result { + Ok(Some(listeners)) => { + generation = generation.saturating_add(1); + startup_runtime_sources::record_tls_generation("rustfs_server_reload_loop", generation); + startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "ok", None, Some(generation)); + info!( + event = "tls_reload_state", + component = LOG_COMPONENT_TLS, + subsystem = LOG_SUBSYSTEM_TLS, + state = "reloaded", + generation, + listeners, + "TLS material updated" + ); + } + Ok(None) => {} + Err(error) => { + startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "rejected", None, None); + warn!(event = "tls_reload_failed", component = LOG_COMPONENT_TLS, subsystem = LOG_SUBSYSTEM_TLS, + result = "rejected", error = %error, "TLS material update rejected"); } } - })) + #[cfg(test)] + group.pass_completed.notify_one(); + } } #[cfg(test)] mod tests { use super::*; use rcgen::CertifiedKey; + use rustfs_config::RUSTFS_TLS_CERT; use std::fs; use std::sync::Once; use tempfile::TempDir; @@ -839,6 +1063,507 @@ mod tests { fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap(); } + #[tokio::test] + #[serial_test::serial] + async fn prepared_generation_shares_crypto_and_serves_sni_with_mtls() { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars([(ENV_SERVER_MTLS_ENABLE, Some("1"))], async { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "default.local"); + let sni_dir = dir.path().join("sni.example"); + fs::create_dir(&sni_dir).expect("SNI directory"); + write_test_cert_pair(&sni_dir, "sni.example"); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).expect("CA parameters"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca = ca_params.self_signed(&ca_key).expect("CA certificate"); + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca.pem()).expect("client trust"); + let mut client_params = rcgen::CertificateParams::new(vec!["client.example".into()]).expect("client parameters"); + client_params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; + let client_key = rcgen::KeyPair::generate().expect("client key"); + let issuer = rcgen::Issuer::from_params(&ca_params, &ca_key); + let client_cert = client_params.signed_by(&client_key, &issuer).expect("client identity"); + let snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .expect("validated TLS snapshot"); + let before = TLS_PREPARE_COUNTS.with(std::cell::Cell::get); + let prepared = PreparedServerTls::from_snapshot(&snapshot) + .expect("prepare generation") + .expect("server TLS"); + let tcp = prepared.server_config(ServerProtocol::Tcp).expect("TCP config"); + let another_tcp = prepared + .server_config(ServerProtocol::Tcp) + .expect("additional listener config"); + assert!(Arc::ptr_eq(&tcp.cert_resolver, &another_tcp.cert_resolver)); + #[cfg(feature = "http3")] + { + let quic_tls = prepared.server_config(ServerProtocol::Http3).expect("QUIC TLS config"); + assert!(Arc::ptr_eq(&tcp.cert_resolver, &quic_tls.cert_resolver)); + } + let after = TLS_PREPARE_COUNTS.with(std::cell::Cell::get); + assert_eq!( + (after.0 - before.0, after.1 - before.1), + (1, 1), + "client verifier and signing resolver must each be prepared once for all protocols and listeners" + ); + + let RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } = snapshot.server.as_ref().expect("SNI material") else { + panic!("multiple selectors expected"); + }; + let mut roots = rustls::RootCertStore::empty(); + for (certificates, _) in cert_key_pairs.values() { + roots.add(certificates[0].clone()).expect("server trust"); + } + for (name, selector) in [("default.local", "default"), ("sni.example", "sni.example")] { + let expected = cert_key_pairs[selector].0[0].clone(); + let client_tls = rustls::ClientConfig::builder() + .with_root_certificates(roots.clone()) + .with_client_auth_cert( + vec![client_cert.der().clone()], + PrivateKeyDer::try_from(client_key.serialize_der()).expect("client private key"), + ) + .expect("mTLS client"); + let (server_io, client_io) = tokio::io::duplex(65536); + let acceptor = prepared.tcp_acceptor().expect("acceptor").get(); + let connector = tokio_rustls::TlsConnector::from(Arc::new(client_tls.clone())); + let (server, client) = tokio::join!( + acceptor.accept(server_io), + connector.connect(rustls::pki_types::ServerName::try_from(name.to_owned()).expect("server name"), client_io) + ); + let server = server.expect("authenticated server handshake"); + let client = client.expect("authenticated client handshake"); + assert_eq!(client.get_ref().1.peer_certificates().expect("server identity")[0], expected); + assert_eq!(server.get_ref().1.peer_certificates().expect("client identity")[0], *client_cert.der()); + + #[cfg(feature = "http3")] + { + let endpoint = quinn::Endpoint::server( + prepared.http3_config().expect("QUIC config"), + "127.0.0.1:0".parse().expect("address"), + ) + .expect("QUIC server"); + let mut client_tls = client_tls; + client_tls.alpn_protocols = vec![b"h3".to_vec()]; + let crypto = quinn::crypto::rustls::QuicClientConfig::try_from(client_tls).expect("QUIC client TLS"); + let mut client_endpoint = + quinn::Endpoint::client("127.0.0.1:0".parse().expect("address")).expect("QUIC client"); + client_endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(crypto))); + let connection = client_endpoint + .connect(endpoint.local_addr().expect("server address"), name) + .expect("connect"); + let (server, client) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(async { endpoint.accept().await.expect("incoming").await }, connection) + }) + .await + .expect("QUIC handshake deadline"); + let server = server.expect("QUIC mTLS server handshake"); + let client = client.expect("QUIC mTLS client handshake"); + let identity = client + .peer_identity() + .expect("QUIC server identity") + .downcast::>>() + .expect("certificate identity"); + assert_eq!(identity[0], expected); + let identity = server + .peer_identity() + .expect("QUIC client identity") + .downcast::>>() + .expect("certificate identity"); + assert_eq!(identity[0], *client_cert.der()); + client.close(0u32.into(), b"done"); + server.close(0u32.into(), b"done"); + endpoint.close(0u32.into(), b"done"); + client_endpoint.close(0u32.into(), b"done"); + } + } + }) + .await; + } + + #[tokio::test] + async fn candidate_replacement_preserves_ca_components_and_default_names() { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "s3.example.com"); + let ca_a = rcgen::generate_simple_self_signed(vec!["ca-a.example".into()]).expect("CA A"); + let ca_b = rcgen::generate_simple_self_signed(vec!["ca-b.example".into()]).expect("CA B"); + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca_a.cert.pem()).expect("primary client CA"); + fs::write(dir.path().join(RUSTFS_CA_CERT), ca_b.cert.pem()).expect("fallback CA"); + let source = TlsSource::from_directory(dir.path()); + let accepted = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("accepted generation"); + fs::remove_file(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME)).expect("remove primary client CA"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("fallback generation parses"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "a fallback must not replace accepted client trust" + ); + + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca_a.cert.pem()).expect("restore primary CA"); + write_test_cert_pair(dir.path(), "wrong.example.com"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("valid but wrong SAN"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "default SAN coverage must survive renewal" + ); + + write_test_cert_pair(dir.path(), "s3.example.com"); + let renewed = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("valid renewal"); + validate_candidate_replacement(&accepted, &renewed).expect("renewal preserving names and trust sources"); + } + + #[tokio::test] + async fn candidate_default_names_cannot_move_into_another_sni_certificate() { + let dir = TempDir::new().expect("TLS directory"); + let sni = dir.path().join("example.com"); + fs::create_dir(&sni).expect("SNI directory"); + write_test_cert_pair(dir.path(), "default.local"); + write_test_cert_pair(&sni, "example.com"); + let source = TlsSource::from_directory(dir.path()); + let accepted = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("accepted selectors"); + write_test_cert_pair(dir.path(), "wrong.local"); + let relocated = rcgen::generate_simple_self_signed(vec!["example.com".into(), "default.local".into()]) + .expect("certificate with relocated SAN"); + fs::write(sni.join(RUSTFS_TLS_CERT), relocated.cert.pem()).expect("SNI certificate"); + fs::write(sni.join(rustfs_config::RUSTFS_TLS_KEY), relocated.signing_key.serialize_pem()).expect("SNI key"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("individually valid candidate"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "another selector's SAN does not cover the default resolver's hostname" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn snapshot_leaf_trust_is_encoded_in_canonical_der_order() { + let dir = TempDir::new().expect("TLS directory"); + let a = rcgen::generate_simple_self_signed(vec!["a.example".into()]).expect("certificate A"); + let b = rcgen::generate_simple_self_signed(vec!["b.example".into()]).expect("certificate B"); + let (first, second) = if a.cert.der().as_ref() > b.cert.der().as_ref() { + (&a, &b) + } else { + (&b, &a) + }; + fs::write(dir.path().join(RUSTFS_TLS_CERT), format!("{}{}", first.cert.pem(), second.cert.pem())) + .expect("descending DER chain"); + fs::write(dir.path().join(rustfs_config::RUSTFS_TLS_KEY), first.signing_key.serialize_pem()).expect("leaf key"); + let mut snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .expect("validated material"); + temp_env::async_with_vars( + [ + (ENV_TRUST_LEAF_CERT_AS_CA, Some("1")), + (ENV_TRUST_SYSTEM_CA, Some("0")), + (ENV_MTLS_CLIENT_CERT, None), + (ENV_MTLS_CLIENT_KEY, None), + ], + async { + enrich_outbound(&mut snapshot).await.expect("derive leaf trust"); + let mut reader = Cursor::new(&snapshot.outbound.root_ca_pem); + let certificates = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .expect("derived certificate PEM"); + assert_eq!( + certificates, + vec![second.cert.der().clone(), first.cert.der().clone()], + "leaf trust must not depend on source enumeration order" + ); + }, + ) + .await; + } + + #[tokio::test] + async fn client_identity_override_rejects_mismatch_and_missing_files() { + let dir = TempDir::new().expect("identity directory"); + let a = rcgen::generate_simple_self_signed(vec!["client-a.example".into()]).expect("identity A"); + let b = rcgen::generate_simple_self_signed(vec!["client-b.example".into()]).expect("identity B"); + let cert = dir.path().join(RUSTFS_CLIENT_CERT_FILENAME); + let key = dir.path().join(RUSTFS_CLIENT_KEY_FILENAME); + fs::write(&cert, a.cert.pem()).expect("client cert"); + fs::write(&key, b.signing_key.serialize_pem()).expect("mismatched key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .is_err() + ); + fs::remove_file(&key).expect("remove override key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .is_err() + ); + fs::write(&key, a.signing_key.serialize_pem()).expect("matching key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .expect("valid override") + .is_some() + ); + } + + enum StartupRotation { + Outbound, + RootsOnly, + IdentityOnly, + ServerOnly, + MissingRoots, + PartialRoots, + MissingIdentity, + } + + async fn assert_initial_outbound_reconciliation(rotation: StartupRotation) { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (ENV_TRUST_SYSTEM_CA, Some("0")), + (ENV_MTLS_CLIENT_CERT, None), + (ENV_MTLS_CLIENT_KEY, None), + (ENV_RUSTFS_EXTRA_CA_CERT, None), + (super::super::tls_publish::ENV_TLS_PUBLISH_DIR, None), + ], + async { + let previous = startup_runtime_sources::current_outbound_tls_state().await; + let previous_files = crate::startup_tls_material::replace_startup_material_files(None); + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "server.example"); + let identity_a = rcgen::generate_simple_self_signed(vec!["client-a.example".into()]).expect("identity A"); + fs::write(dir.path().join(RUSTFS_CA_CERT), identity_a.cert.pem()).expect("trust A"); + fs::write(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME), identity_a.cert.pem()).expect("client certificate A"); + fs::write(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME), identity_a.signing_key.serialize_pem()) + .expect("client key A"); + if matches!(rotation, StartupRotation::PartialRoots) { + let public_ca = + rcgen::generate_simple_self_signed(vec!["public-ca.example".into()]).expect("distinct public CA"); + fs::write(dir.path().join(rustfs_config::RUSTFS_PUBLIC_CERT), public_ca.cert.pem()) + .expect("additional public trust"); + } + let path = dir.path().to_str().expect("UTF8 path").to_string(); + let mut config = crate::config::Config::new("127.0.0.1:0", Vec::new()); + config.tls_path = Some(path.clone()); + crate::startup_tls_material::init_outbound_tls_material(&config) + .await + .expect("real startup publication"); + let startup = startup_runtime_sources::current_outbound_tls_state().await; + + write_test_cert_pair(dir.path(), "server.example"); + match rotation { + StartupRotation::Outbound | StartupRotation::RootsOnly | StartupRotation::IdentityOnly => { + let identity_b = rcgen::generate_simple_self_signed(vec!["client-b.example".into()]).expect("identity B"); + if !matches!(rotation, StartupRotation::IdentityOnly) { + fs::write(dir.path().join(RUSTFS_CA_CERT), identity_b.cert.pem()).expect("trust B"); + } + if !matches!(rotation, StartupRotation::RootsOnly) { + fs::write(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME), identity_b.cert.pem()) + .expect("client certificate B"); + fs::write(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME), identity_b.signing_key.serialize_pem()) + .expect("client key B"); + } + } + StartupRotation::ServerOnly => {} + StartupRotation::MissingRoots | StartupRotation::PartialRoots => { + fs::remove_file(dir.path().join(RUSTFS_CA_CERT)).expect("remove startup roots"); + } + StartupRotation::MissingIdentity => { + fs::remove_file(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME)).expect("remove startup client certificate"); + fs::remove_file(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME)).expect("remove startup client key"); + } + } + let initial = load_tls_material(&path).await.expect("HTTP initialization snapshot"); + let incomplete = matches!( + rotation, + StartupRotation::MissingRoots | StartupRotation::PartialRoots | StartupRotation::MissingIdentity + ); + let expected_roots = if incomplete { + startup.root_ca_pem.clone().expect("startup trust") + } else { + initial.outbound.root_ca_pem.clone() + }; + let expected_identity = if incomplete { + startup.mtls_identity.clone() + } else { + initial.outbound.mtls_identity.clone() + }; + let prepared = PreparedServerTls::from_snapshot(&initial).expect("config B").expect("TLS B"); + let holder = prepared.tcp_acceptor().expect("TLS B acceptor"); + let initial_acceptor = holder.get(); + let task = spawn_reload_loop( + path.clone(), + holder.clone(), + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("register HTTP initialization snapshot") + .expect("reload enabled"); + let group = RELOAD_GROUPS + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(&path) + .and_then(Weak::upgrade) + .expect("registered reload group"); + notify_server_publication(&path); + tokio::time::timeout(Duration::from_secs(10), group.pass_completed.notified()) + .await + .expect("first reload pass completed"); + let first = startup_runtime_sources::current_outbound_tls_state().await; + notify_server_publication(&path); + tokio::time::timeout(Duration::from_secs(10), group.pass_completed.notified()) + .await + .expect("second reload pass completed"); + let second = startup_runtime_sources::current_outbound_tls_state().await; + + task.abort(); + let _ = task.await; + tokio::time::timeout(Duration::from_secs(10), async { + while RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&path) { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload group exits before restoring global state"); + startup_runtime_sources::publish_outbound_tls_state( + TlsGeneration(second.generation.0.saturating_add(1)), + &rustfs_tls_runtime::OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }, + ) + .await; + crate::startup_tls_material::replace_startup_material_files(previous_files); + + assert_eq!(first.root_ca_pem.as_deref(), Some(expected_roots.as_slice())); + assert!( + first.mtls_identity == expected_identity, + "published identity must match the initialized listener snapshot" + ); + let rotate_outbound = matches!( + rotation, + StartupRotation::Outbound | StartupRotation::RootsOnly | StartupRotation::IdentityOnly + ); + let expected_generation = startup.generation.0.saturating_add(u64::from(rotate_outbound)); + assert_eq!(first.generation.0, expected_generation); + assert_eq!(second.generation, first.generation, "unchanged material must not churn outbound clients"); + assert!( + rotate_outbound || Arc::ptr_eq(&holder.get(), &initial_acceptor), + "non-publications must retain the initial server configuration" + ); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_reconciles_initial_outbound_after_startup_rotation() { + for rotation in [ + StartupRotation::RootsOnly, + StartupRotation::IdentityOnly, + StartupRotation::Outbound, + ] { + assert_initial_outbound_reconciliation(rotation).await; + } + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_server_rotation_preserves_matching_outbound_generation() { + assert_initial_outbound_reconciliation(StartupRotation::ServerOnly).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_missing_roots_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::MissingRoots).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_partial_missing_roots_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::PartialRoots).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_missing_identity_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::MissingIdentity).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn late_listener_joins_accepted_generation_when_source_is_invalid() { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars([(ENV_TLS_RELOAD_ENABLE, Some("1")), (ENV_SERVER_MTLS_ENABLE, Some("0"))], async { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "server.example"); + let source = TlsSource::from_directory(dir.path()); + let a = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("generation A"); + let prepared_a = PreparedServerTls::from_snapshot(&a).expect("prepare A").expect("TLS A"); + let first = prepared_a.tcp_acceptor().expect("config A"); + let path = dir.path().to_str().expect("UTF8 path").to_string(); + let task_a = spawn_reload_loop( + path.clone(), + first.clone(), + a, + prepared_a, + #[cfg(feature = "http3")] + None, + ) + .expect("register A") + .expect("reload enabled"); + // No async yield between producing B and registration: the actor cannot accept B. + write_test_cert_pair(dir.path(), "server.example"); + let b = rustfs_tls_runtime::TlsMaterialSnapshot::load_from_snapshot(&source.snapshot().expect("pin B")) + .expect("generation B"); + let prepared_b = PreparedServerTls::from_snapshot(&b).expect("prepare B").expect("TLS B"); + let second = prepared_b.tcp_acceptor().expect("config B"); + assert!(!Arc::ptr_eq(&first.get(), &second.get())); + fs::write(dir.path().join(RUSTFS_TLS_CERT), "invalid certificate").expect("break source"); + let task_b = spawn_reload_loop( + path.clone(), + second.clone(), + b, + prepared_b, + #[cfg(feature = "http3")] + None, + ) + .expect("register B") + .expect("reload enabled"); + assert!( + Arc::ptr_eq(&first.get(), &second.get()), + "late listener must use the group's accepted config" + ); + task_a.abort(); + task_b.abort(); + let _ = task_a.await; + let _ = task_b.await; + tokio::time::timeout(Duration::from_secs(5), async { + while RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&path) { + tokio::task::yield_now().await; + } + }) + .await + .expect("group exits when all registrations are dropped"); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn oidc_extra_ca_cert_loads_configured_bundle() { @@ -1070,15 +1795,17 @@ mod tests { ensure_rustls_crypto_provider(); let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate"); - let config = build_server_config( - ServerCertSource::SingleCert { + let config = PreparedServerTls::from_material( + Some(&RuntimeServerTlsMaterial::SingleCert { certs: vec![cert.cert.der().clone()], key: rustls::pki_types::PrivateKeyDer::try_from(cert.signing_key.serialize_der()) .expect("private key should convert"), - }, + }), None, - ServerProtocol::Http3, ) + .expect("prepare TLS") + .expect("server certificate") + .server_config(ServerProtocol::Http3) .expect("HTTP/3 TLS config should build"); assert_eq!(config.alpn_protocols, vec![b"h3".to_vec()]); diff --git a/rustfs/src/server/tls_publish.rs b/rustfs/src/server/tls_publish.rs new file mode 100644 index 000000000..ab11787e9 --- /dev/null +++ b/rustfs/src/server/tls_publish.rs @@ -0,0 +1,1341 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Durable server-leaf publication. Trust and client identity remain owned by their original source. + +use super::tls_material; +use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_ENABLE, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustfs_tls_runtime::{ServerTlsMaterial, TlsMaterialSnapshot, TlsSource}; +use rustls::pki_types::{CertificateDer, pem::PemObject}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Component, Path, PathBuf}, +}; +use uuid::Uuid; + +pub(crate) const ENV_TLS_PUBLISH_DIR: &str = "RUSTFS_TLS_PUBLISH_DIR"; +pub(crate) const SCHEMA: &str = "rustfs.tls-publish/v1alpha1"; +pub(crate) const RESOURCE: &str = "server-default"; +const MAX_CERT: usize = 64 * 1024; +const MAX_KEY: usize = 32 * 1024; +const MANIFEST: &str = "manifest.json"; +const CURRENT: &str = "server-current"; +const INITIALIZED: &str = "initialized"; +const INITIALIZED_CONTENT: &[u8] = b"rustfs.tls-publish/v1alpha1\n"; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct OperationReads { + certificates: usize, + keys: usize, + manifests: usize, + manifest_decodes: usize, + certificate_pem_traversals: usize, + leaf_der_decodes: usize, +} + +#[cfg(test)] +thread_local! { + static OPERATION_READS: std::cell::Cell = const { std::cell::Cell::new(OperationReads { + certificates: 0, keys: 0, manifests: 0, manifest_decodes: 0, + certificate_pem_traversals: 0, leaf_der_decodes: 0, + }) }; +} + +#[cfg(test)] +fn count_operation(action: impl FnOnce(&mut OperationReads)) { + OPERATION_READS.with(|counts| { + let mut value = counts.get(); + action(&mut value); + counts.set(value); + }); +} + +fn certificate_pem_items( + bytes: &[u8], +) -> impl Iterator, rustls::pki_types::pem::Error>> + '_ { + #[cfg(test)] + count_operation(|counts| counts.certificate_pem_traversals += 1); + CertificateDer::pem_slice_iter(bytes) +} + +#[cfg(test)] +#[derive(Clone, Copy)] +enum FailurePoint { + None, + AfterPointerRename, + AfterPointerRenameUnreadable, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PublishRequest { + pub schema_version: String, + pub resource_id: String, + pub expected_target_revision: String, + pub issuance_id: Uuid, + pub certificate_pem: String, + pub private_key_pem: String, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + schema_version: String, + issuance_id: Uuid, + target_revision: Uuid, + certificate_fingerprint: String, + not_after_unix: i64, + payload_digest: String, +} + +// A disk operation pins one immutable generation and verifies its envelope once. +// This value is never retained across requests or reload ticks. +struct PublishedGeneration { + manifest: Manifest, + chain: Vec>, + leaf: LeafMetadata, + key_pem: Vec, +} + +struct LeafMetadata { + fingerprint: String, + not_after_unix: i64, + names: std::collections::BTreeSet, + is_ca: bool, +} + +#[derive(Serialize)] +pub(crate) struct CertificateState { + pub schema_version: &'static str, + pub resource_id: &'static str, + pub target_revision: String, + pub certificate_fingerprint: String, + pub not_after_unix: i64, + pub last_issuance_id: Option, + pub observed_certificate_fingerprint: Option, +} + +#[derive(Serialize)] +pub(crate) struct PublishReceipt { + pub schema_version: &'static str, + pub resource_id: &'static str, + pub issuance_id: Uuid, + pub target_revision: String, + pub certificate_fingerprint: String, + pub not_after_unix: i64, + pub observed_certificate_fingerprint: Option, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PublishError { + #[error("certificate publication is disabled or not configured safely")] + Disabled, + #[error("invalid certificate publication request or source")] + Invalid, + #[error("certificate publication revision conflict")] + Conflict, + #[error("certificate publication is busy; retry the identical issuance")] + Busy, + #[error("certificate publication storage failed; retry the identical issuance")] + Storage, +} + +impl From for PublishError { + fn from(_: std::io::Error) -> Self { + Self::Storage + } +} + +fn hex_digest(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn payload_digest(certificate: &[u8], key: &[u8]) -> String { + let mut digest = Sha256::new(); + // Both inputs have bounded sizes; the fixed-width encoding survives architecture changes. + digest.update((certificate.len() as u128).to_be_bytes()); + digest.update(certificate); + digest.update((key.len() as u128).to_be_bytes()); + digest.update(key); + digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn parse_leaf(leaf: &[u8]) -> Result { + #[cfg(test)] + count_operation(|counts| counts.leaf_der_decodes += 1); + let (remaining, parsed) = x509_parser::parse_x509_certificate(leaf).map_err(|_| PublishError::Invalid)?; + if !remaining.is_empty() { + return Err(PublishError::Invalid); + } + let mut names = std::collections::BTreeSet::new(); + if let Some(san) = parsed.subject_alternative_name().map_err(|_| PublishError::Invalid)? { + for name in &san.value.general_names { + match name { + x509_parser::extensions::GeneralName::DNSName(name) => { + names.insert(format!("dns:{}", name.to_ascii_lowercase())); + } + x509_parser::extensions::GeneralName::IPAddress(bytes) => { + names.insert(format!("ip:{bytes:?}")); + } + _ => {} + } + } + } + Ok(LeafMetadata { + fingerprint: hex_digest(leaf), + not_after_unix: parsed.validity().not_after.timestamp(), + names, + is_ca: parsed + .basic_constraints() + .map_err(|_| PublishError::Invalid)? + .is_some_and(|constraints| constraints.value.ca), + }) +} + +fn snapshot_leaf(snapshot: &TlsMaterialSnapshot) -> Option { + match &snapshot.server { + Some(ServerTlsMaterial::SingleCert { certs, .. }) => certs.first().map(|cert| hex_digest(cert.as_ref())), + _ => None, + } +} + +fn observed(tls_path: &str, expected_chain: &[CertificateDer<'_>]) -> Option { + let snapshot = tls_material::observed_server_snapshot(tls_path)?; + match &snapshot.server { + Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == expected_chain => snapshot_leaf(&snapshot), + _ => None, + } +} + +fn open_regular(path: &Path, writable: bool, create: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true).write(writable).create(create); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(PublishError::Invalid); + } + Ok(file) +} + +fn read_limited(path: &Path, limit: usize) -> Result, PublishError> { + let file = open_regular(path, false, false)?; + #[cfg(test)] + count_operation(|counts| match path.file_name().and_then(|name| name.to_str()) { + Some(RUSTFS_TLS_CERT) => counts.certificates += 1, + Some(RUSTFS_TLS_KEY) => counts.keys += 1, + Some(MANIFEST) => counts.manifests += 1, + _ => {} + }); + let mut bytes = Vec::new(); + file.take(u64::try_from(limit).map_err(|_| PublishError::Invalid)? + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > limit { + return Err(PublishError::Invalid); + } + Ok(bytes) +} + +fn validate_root(root: &Path) -> Result<(), PublishError> { + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PublishError::Disabled); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.mode() & 0o077 != 0 || metadata.uid() != rustix::process::geteuid().as_raw() { + return Err(PublishError::Disabled); + } + } + #[cfg(not(unix))] + { + Err(PublishError::Disabled) + } + #[cfg(unix)] + { + Ok(()) + } +} + +fn configured_root() -> Result, PublishError> { + let root = rustfs_utils::get_env_opt_str(ENV_TLS_PUBLISH_DIR).map(PathBuf::from); + if let Some(root) = &root { + validate_root(root)?; + } + Ok(root) +} + +fn single_default_source(tls_path: &str) -> Result<(), PublishError> { + if fs::symlink_metadata(Path::new(tls_path).join("..data")).is_ok() { + return Err(PublishError::Disabled); + } + for entry in fs::read_dir(tls_path)? { + let entry = entry?; + if !entry.file_name().to_string_lossy().starts_with('.') && entry.path().is_dir() { + return Err(PublishError::Invalid); + } + } + Ok(()) +} + +fn current_generation(root: &Path) -> Result, PublishError> { + let initialized = match fs::symlink_metadata(root.join(INITIALIZED)) { + Ok(_) => { + if read_limited(&root.join(INITIALIZED), 128)? != INITIALIZED_CONTENT { + return Err(PublishError::Invalid); + } + true + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => return Err(error.into()), + }; + let pointer = root.join(CURRENT); + match fs::symlink_metadata(&pointer) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return if initialized { Err(PublishError::Invalid) } else { Ok(None) }; + } + Err(error) => return Err(error.into()), + Ok(metadata) if !metadata.file_type().is_symlink() => return Err(PublishError::Invalid), + _ => {} + } + let target = fs::read_link(pointer)?; + let mut components = target.components(); + if components.next() != Some(Component::Normal("generations".as_ref())) { + return Err(PublishError::Invalid); + } + let Some(Component::Normal(version)) = components.next() else { + return Err(PublishError::Invalid); + }; + let id = version + .to_str() + .and_then(|text| Uuid::parse_str(text).ok()) + .ok_or(PublishError::Invalid)?; + if components.next().is_some() || version != id.to_string().as_str() { + return Err(PublishError::Invalid); + } + let generations = root.join("generations"); + if fs::symlink_metadata(&generations)?.file_type().is_symlink() { + return Err(PublishError::Invalid); + } + let directory = root.join(target); + if !fs::symlink_metadata(&directory)?.is_dir() { + return Err(PublishError::Invalid); + } + PublishedGeneration::load(directory, id).map(Some) +} + +fn validate_source_separation(root: &Path, tls_path: &str) -> Result<(), PublishError> { + let root = fs::canonicalize(root)?; + let source = fs::canonicalize(tls_path)?; + if root.starts_with(&source) || source.starts_with(&root) { + return Err(PublishError::Disabled); + } + Ok(()) +} + +impl PublishedGeneration { + fn into_snapshot(self, source: &rustfs_tls_runtime::TlsSourceSnapshot) -> Result { + TlsMaterialSnapshot::load_from_snapshot_with_server_identity(source, self.chain, &self.key_pem) + .map_err(|_| PublishError::Invalid) + } + + fn load(directory: PathBuf, revision: Uuid) -> Result { + let manifest_bytes = read_limited(&directory.join(MANIFEST), 4096)?; + #[cfg(test)] + count_operation(|counts| counts.manifest_decodes += 1); + let manifest: Manifest = serde_json::from_slice(&manifest_bytes).map_err(|_| PublishError::Invalid)?; + let certificate = read_limited(&directory.join(RUSTFS_TLS_CERT), MAX_CERT)?; + let key = read_limited(&directory.join(RUSTFS_TLS_KEY), MAX_KEY)?; + let chain = certificate_pem_items(&certificate) + .collect::, _>>() + .map_err(|_| PublishError::Invalid)?; + let leaf = parse_leaf(chain.first().ok_or(PublishError::Invalid)?.as_ref())?; + if manifest.schema_version != SCHEMA + || manifest.issuance_id.is_nil() + || manifest.target_revision.is_nil() + || manifest.target_revision != revision + || manifest.certificate_fingerprint != leaf.fingerprint + || manifest.not_after_unix != leaf.not_after_unix + || manifest.payload_digest != payload_digest(&certificate, &key) + { + return Err(PublishError::Invalid); + } + Ok(Self { + manifest, + chain, + leaf, + key_pem: key, + }) + } +} + +#[cfg(test)] +fn current_directory(root: &Path) -> Result, PublishError> { + current_generation(root).map(|generation| { + generation.map(|generation| root.join("generations").join(generation.manifest.target_revision.to_string())) + }) +} + +pub(crate) fn configured_server_snapshot( + tls_path: &str, + source: &rustfs_tls_runtime::TlsSourceSnapshot, +) -> Result, PublishError> { + let Some(root) = configured_root()? else { + return Ok(None); + }; + validate_source_separation(&root, tls_path)?; + single_default_source(tls_path)?; + current_generation(&root)? + .map(|generation| generation.into_snapshot(source)) + .transpose() +} + +fn state_at(root: &Path, tls_path: &str) -> Result { + let generation = current_generation(root)?; + state_from_generation(generation.as_ref(), tls_path) +} + +fn state_from_generation(generation: Option<&PublishedGeneration>, tls_path: &str) -> Result { + if let Some(generation) = generation { + return Ok(CertificateState { + schema_version: SCHEMA, + resource_id: RESOURCE, + target_revision: generation.manifest.target_revision.to_string(), + certificate_fingerprint: generation.leaf.fingerprint.clone(), + not_after_unix: generation.leaf.not_after_unix, + last_issuance_id: Some(generation.manifest.issuance_id), + observed_certificate_fingerprint: observed(tls_path, &generation.chain), + }); + } + let snapshot = tls_material::observed_server_snapshot(tls_path).ok_or(PublishError::Disabled)?; + let Some(ServerTlsMaterial::SingleCert { certs, .. }) = &snapshot.server else { + return Err(PublishError::Invalid); + }; + let leaf = parse_leaf(certs.first().ok_or(PublishError::Invalid)?.as_ref())?; + Ok(CertificateState { + schema_version: SCHEMA, + resource_id: RESOURCE, + target_revision: format!("bootstrap-{}", leaf.fingerprint), + certificate_fingerprint: leaf.fingerprint, + not_after_unix: leaf.not_after_unix, + last_issuance_id: None, + observed_certificate_fingerprint: observed(tls_path, certs), + }) +} + +pub(crate) async fn certificate_state(tls_path: String) -> Result { + tokio::task::spawn_blocking(move || { + let root = configured_root()?.ok_or(PublishError::Disabled)?; + validate_source_separation(&root, &tls_path)?; + single_default_source(&tls_path)?; + state_at(&root, &tls_path) + }) + .await + .map_err(|_| PublishError::Storage)? +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<(), PublishError> { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + Ok(()) +} + +fn receipt(state: CertificateState, issuance_id: Uuid) -> PublishReceipt { + PublishReceipt { + schema_version: SCHEMA, + resource_id: RESOURCE, + issuance_id, + target_revision: state.target_revision, + certificate_fingerprint: state.certificate_fingerprint, + not_after_unix: state.not_after_unix, + observed_certificate_fingerprint: state.observed_certificate_fingerprint, + } +} + +fn mark_initialized(root: &Path) -> Result<(), PublishError> { + let marker = root.join(INITIALIZED); + if fs::symlink_metadata(&marker).is_ok() { + if read_limited(&marker, 128)? != INITIALIZED_CONTENT { + return Err(PublishError::Invalid); + } + } else { + // Partial writes must remain unreferenced so an identical issuance can recover after I/O failure. + let temporary = root.join(format!(".initialized-{}", Uuid::new_v4())); + write_new(&temporary, INITIALIZED_CONTENT)?; + fs::rename(temporary, &marker)?; + } + File::open(root)?.sync_all()?; + Ok(()) +} + +fn publish_at(root: &Path, tls_path: &str, request: PublishRequest) -> Result { + publish_at_inner( + root, + tls_path, + request, + #[cfg(test)] + FailurePoint::None, + ) +} + +fn publish_at_inner( + root: &Path, + tls_path: &str, + request: PublishRequest, + #[cfg(test)] failure: FailurePoint, +) -> Result { + validate_root(root)?; + validate_source_separation(root, tls_path)?; + single_default_source(tls_path)?; + if request.schema_version != SCHEMA + || request.resource_id != RESOURCE + || request.issuance_id.is_nil() + || request.certificate_pem.is_empty() + || request.certificate_pem.len() > MAX_CERT + || request.private_key_pem.is_empty() + || request.private_key_pem.len() > MAX_KEY + || request.expected_target_revision.len() > 128 + { + return Err(PublishError::Invalid); + } + // OS lock is shared by processes using this local volume. Captured CAS tokens are never refreshed on retry. + let lock = open_regular(&root.join("publisher.lock"), true, true)?; + lock.try_lock().map_err(|error| match error { + std::fs::TryLockError::WouldBlock => PublishError::Busy, + std::fs::TryLockError::Error(_) => PublishError::Storage, + })?; + let current = current_generation(root)?; + let state = state_from_generation(current.as_ref(), tls_path)?; + let digest = payload_digest(request.certificate_pem.as_bytes(), request.private_key_pem.as_bytes()); + if state.last_issuance_id == Some(request.issuance_id) { + let generation = current.as_ref().ok_or(PublishError::Invalid)?; + if generation.manifest.payload_digest != digest { + return Err(PublishError::Conflict); + } + // A prior response may have been lost after rename or directory sync. + File::open(root)?.sync_all()?; + mark_initialized(root)?; + return Ok(receipt(state, request.issuance_id)); + } + if request.expected_target_revision != state.target_revision { + return Err(PublishError::Conflict); + } + let accepted = tls_material::observed_server_snapshot(tls_path).ok_or(PublishError::Disabled)?; + let Some(ServerTlsMaterial::SingleCert { certs, .. }) = &accepted.server else { + return Err(PublishError::Invalid); + }; + let old_names = if let Some(generation) = current.as_ref() { + generation.leaf.names.clone() + } else { + if snapshot_leaf(&accepted).as_ref() != Some(&state.certificate_fingerprint) { + return Err(PublishError::Conflict); + } + parse_leaf(certs.first().ok_or(PublishError::Invalid)?.as_ref())?.names + }; + let new_chain = certificate_pem_items(request.certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|_| PublishError::Invalid)?; + let new_leaf = parse_leaf(new_chain.first().ok_or(PublishError::Invalid)?.as_ref())?; + if new_leaf.is_ca || !old_names.is_subset(&new_leaf.names) { + return Err(PublishError::Invalid); + } + let generations = root.join("generations"); + if !generations.exists() { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&generations)?; + File::open(root)?.sync_all()?; + } + if !fs::symlink_metadata(&generations)?.is_dir() { + return Err(PublishError::Invalid); + } + let revision = Uuid::new_v4(); + let directory = generations.join(revision.to_string()); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&directory)?; + let mut pointer_may_reference = false; + let result = (|| { + write_new(&directory.join(RUSTFS_TLS_CERT), request.certificate_pem.as_bytes())?; + write_new(&directory.join(RUSTFS_TLS_KEY), request.private_key_pem.as_bytes())?; + let source = TlsSource::from_directory(tls_path); + let candidate = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &source.snapshot().map_err(|_| PublishError::Invalid)?, + Some(&directory), + ) + .map_err(|_| PublishError::Invalid)?; + if !matches!(&candidate.server, Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == &new_chain) { + return Err(PublishError::Invalid); + } + tls_material::validate_candidate_replacement(&accepted, &candidate).map_err(|_| PublishError::Invalid)?; + tls_material::validate_required_material_files( + &crate::startup_tls_material::required_startup_material_files(tls_path), + &candidate, + ) + .map_err(|_| PublishError::Invalid)?; + let prepared = tls_material::PreparedServerTls::from_snapshot(&candidate) + .map_err(|_| PublishError::Invalid)? + .ok_or(PublishError::Invalid)?; + prepared.tcp_acceptor().map_err(|_| PublishError::Invalid)?; + #[cfg(feature = "http3")] + prepared.http3_config().map_err(|_| PublishError::Invalid)?; + let ledger = Manifest { + schema_version: SCHEMA.into(), + issuance_id: request.issuance_id, + target_revision: revision, + certificate_fingerprint: new_leaf.fingerprint.clone(), + not_after_unix: new_leaf.not_after_unix, + payload_digest: digest, + }; + write_new( + &directory.join(MANIFEST), + &serde_json::to_vec(&ledger).map_err(|_| PublishError::Invalid)?, + )?; + File::open(&directory)?.sync_all()?; + File::open(&generations)?.sync_all()?; + let temporary = root.join(format!(".pointer-{revision}")); + #[cfg(unix)] + std::os::unix::fs::symlink(Path::new("generations").join(revision.to_string()), &temporary)?; + #[cfg(not(unix))] + let () = Err::<(), _>(PublishError::Disabled)?; + // A rename error can itself have an uncertain outcome on a failing filesystem. + pointer_may_reference = true; + fs::rename(&temporary, root.join(CURRENT))?; + #[cfg(test)] + { + if matches!(failure, FailurePoint::AfterPointerRenameUnreadable) { + fs::remove_file(root.join(CURRENT))?; + write_new(&root.join(CURRENT), b"unreadable pointer")?; + } + if !matches!(failure, FailurePoint::None) { + return Err(PublishError::Storage); + } + } + File::open(root)?.sync_all()?; + mark_initialized(root)?; + let installed = PublishedGeneration { + manifest: ledger, + chain: new_chain, + leaf: new_leaf, + key_pem: request.private_key_pem.into_bytes(), + }; + Ok(receipt(state_from_generation(Some(&installed), tls_path)?, request.issuance_id)) + })(); + // Observation errors must never convert an uncertain publication into deletion of serving material. + if result.is_err() && !pointer_may_reference { + let _ = fs::remove_dir_all(&directory); + } + result +} + +pub(crate) async fn publish(tls_path: String, request: PublishRequest) -> Result { + if !rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE) { + return Err(PublishError::Disabled); + } + if rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, rustfs_config::DEFAULT_TRUST_LEAF_CERT_AS_CA) { + return Err(PublishError::Disabled); + } + let root = configured_root()?.ok_or(PublishError::Disabled)?; + let notification_path = tls_path.clone(); + let result = tokio::task::spawn_blocking(move || publish_at(&root, &tls_path, request)) + .await + .map_err(|_| PublishError::Storage)?; + tls_material::notify_server_publication(¬ification_path); + result +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn pair(subject: &str) -> rcgen::CertifiedKey { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + rcgen::generate_simple_self_signed(vec![subject.into()]).expect("certificate") + } + + fn build_request(expected: &str, certificate: &rcgen::CertifiedKey) -> PublishRequest { + PublishRequest { + schema_version: SCHEMA.into(), + resource_id: RESOURCE.into(), + expected_target_revision: expected.into(), + issuance_id: Uuid::new_v4(), + certificate_pem: certificate.cert.pem(), + private_key_pem: certificate.signing_key.serialize_pem(), + } + } + + fn clone_request(request: &PublishRequest) -> PublishRequest { + PublishRequest { + schema_version: request.schema_version.clone(), + resource_id: request.resource_id.clone(), + expected_target_revision: request.expected_target_revision.clone(), + issuance_id: request.issuance_id, + certificate_pem: request.certificate_pem.clone(), + private_key_pem: request.private_key_pem.clone(), + } + } + + fn measure_operation(operation: impl FnOnce() -> T) -> (T, OperationReads) { + OPERATION_READS.with(|counts| counts.set(OperationReads::default())); + let result = operation(); + (result, OPERATION_READS.with(std::cell::Cell::get)) + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_publication_rejects_missing_startup_ca_source() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (rustfs_config::ENV_TRUST_SYSTEM_CA, Some("0")), + (rustfs_config::ENV_MTLS_CLIENT_CERT, None), + (rustfs_config::ENV_MTLS_CLIENT_KEY, None), + (rustfs_config::ENV_RUSTFS_EXTRA_CA_CERT, None), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let previous = crate::startup_runtime_sources::current_outbound_tls_state().await; + let previous_files = crate::startup_tls_material::replace_startup_material_files(None); + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + fs::write(base.path().join(rustfs_config::RUSTFS_PUBLIC_CERT), pair("public-ca.example").cert.pem()) + .expect("public trust"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), pair("fallback-ca.example").cert.pem()) + .expect("fallback trust"); + let path = base.path().to_str().expect("path"); + let mut config = crate::config::Config::new("127.0.0.1:0", Vec::new()); + config.tls_path = Some(path.into()); + crate::startup_tls_material::init_outbound_tls_material(&config) + .await + .expect("real startup publication"); + fs::remove_file(base.path().join(rustfs_config::RUSTFS_CA_CERT)).expect("remove startup CA before listener"); + let initial = tls_material::load_tls_material(path).await.expect("HTTP initial snapshot"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload enabled"); + let state = state_at(root.path(), path).expect("valid bootstrap CAS state"); + let result = publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))); + let installed = current_directory(root.path()).expect("read durable pointer"); + task.abort(); + let _ = task.await; + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while tls_material::observed_server_snapshot(path).is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload group exits before restoring startup state"); + crate::startup_runtime_sources::publish_outbound_tls_state( + rustfs_tls_runtime::TlsGeneration( + crate::startup_runtime_sources::current_outbound_tls_generation().saturating_add(1), + ), + &rustfs_tls_runtime::OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }, + ) + .await; + crate::startup_tls_material::replace_startup_material_files(previous_files); + assert!( + matches!(result, Err(PublishError::Invalid)), + "source loss must reject an otherwise valid renewal" + ); + assert!(installed.is_none(), "rejected trust must not acquire a durable publication pointer"); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_operation_reads_are_bounded() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let bootstrap = state_at(root.path(), path).expect("bootstrap"); + let request = build_request(&bootstrap.target_revision, &pair("server.example")); + let retry = clone_request(&request); + publish_at(root.path(), path, request).expect("first publication"); + let (state, get_counts) = measure_operation(|| state_at(root.path(), path).expect("state")); + let (_, retry_counts) = measure_operation(|| publish_at(root.path(), path, retry).expect("retry")); + let (_, reload_counts) = measure_operation(|| { + let generation = current_generation(root.path()).expect("generation").expect("published"); + generation + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .expect("runtime snapshot") + }); + let (_, publish_counts) = measure_operation(|| { + publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))) + .expect("next publication") + }); + println!("GET={get_counts:?}; RETRY={retry_counts:?}; PUBLISH={publish_counts:?}"); + println!("RELOAD={reload_counts:?}"); + assert_eq!( + reload_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + get_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + retry_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + publish_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 2, + leaf_der_decodes: 2 + } + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_preserves_trust_rejects_stale_and_recovers_lost_receipt() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("original certificate"); + fs::rename(base.path().join(RUSTFS_TLS_CERT), base.path().join("seed-certificate.pem")) + .expect("symlink seed backing"); + std::os::unix::fs::symlink("seed-certificate.pem", base.path().join(RUSTFS_TLS_CERT)) + .expect("Certbot-style seed symlink"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("original key"); + let old_ca = pair("ca-old.example"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), old_ca.cert.pem()).expect("trust file"); + let path = base.path().to_str().expect("base path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let bootstrap = state_at(root.path(), path).expect("bootstrap state"); + let candidate = pair("server.example"); + let request = build_request(&bootstrap.target_revision, &candidate); + let retry = clone_request(&request); + let mutated = clone_request(&request); + let receipt = publish_at(root.path(), path, request).expect("durable receipt"); + assert_ne!(receipt.target_revision, bootstrap.target_revision); + assert_eq!(receipt.certificate_fingerprint, hex_digest(candidate.cert.der().as_ref())); + assert_eq!( + fs::read(base.path().join(RUSTFS_TLS_CERT)).expect("base certificate"), + original.cert.pem().as_bytes() + ); + let new_ca = pair("ca-new.example"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), new_ca.cert.pem()) + .expect("independent trust rotation"); + let directory = current_directory(root.path()).expect("pointer").expect("generation"); + let snapshot = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &TlsSource::from_directory(base.path()).snapshot().expect("source"), + Some(&directory), + ) + .expect("overlay snapshot"); + assert_eq!(snapshot.outbound.root_ca_pem, new_ca.cert.pem().as_bytes()); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + let retry_receipt = publish_at(root.path(), path, retry).expect("lost receipt recovery"); + assert_eq!(retry_receipt.target_revision, receipt.target_revision); + let mut mutated = mutated; + mutated.private_key_pem = pair("server.example").signing_key.serialize_pem(); + assert!(matches!(publish_at(root.path(), path, mutated), Err(PublishError::Conflict))); + assert!(matches!( + publish_at(root.path(), path, build_request(&bootstrap.target_revision, &pair("server.example"))), + Err(PublishError::Conflict) + )); + assert_eq!( + state_at(root.path(), path).expect("accepted state").target_revision, + receipt.target_revision + ); + let expanded = rcgen::generate_simple_self_signed(vec!["server.example".into(), "extra.example".into()]) + .expect("expanded names"); + let expanded_receipt = publish_at(root.path(), path, build_request(&receipt.target_revision, &expanded)) + .expect("pending expanded generation"); + assert!( + matches!( + publish_at(root.path(), path, build_request(&expanded_receipt.target_revision, &candidate)), + Err(PublishError::Invalid) + ), + "a later publication must preserve names of the durable generation even before it is observed" + ); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let one = build_request(&expanded_receipt.target_revision, &expanded); + let two = build_request(&expanded_receipt.target_revision, &expanded); + let root_path = root.path(); + let (one, two) = std::thread::scope(|scope| { + let barrier_one = barrier.clone(); + let first = scope.spawn(move || { + barrier_one.wait(); + publish_at(root_path, path, one) + }); + let second = scope.spawn(|| { + barrier.wait(); + publish_at(root_path, path, two) + }); + (first.join().expect("first publisher"), second.join().expect("second publisher")) + }); + assert_eq!(usize::from(one.is_ok()) + usize::from(two.is_ok()), 1, "one concurrent CAS winner"); + assert!( + matches!(one, Err(PublishError::Conflict | PublishError::Busy)) + || matches!(two, Err(PublishError::Conflict | PublishError::Busy)) + ); + let latest = state_at(root.path(), path).expect("concurrent winner"); + let interrupted = build_request(&latest.target_revision, &expanded); + let retry = clone_request(&interrupted); + assert!( + matches!( + publish_at_inner(root.path(), path, interrupted, FailurePoint::AfterPointerRename), + Err(PublishError::Storage) + ), + "failure after rename has an unknown durable outcome" + ); + let recovered = + publish_at(root.path(), path, retry).expect("same issuance recovers and syncs the installed generation"); + assert_ne!(recovered.target_revision, latest.target_revision); + assert!( + current_directory(root.path()) + .expect("installed generation survives error") + .is_some() + ); + let interrupted = build_request(&recovered.target_revision, &expanded); + let before = fs::read_dir(root.path().join("generations")).expect("generations").count(); + assert!(matches!( + publish_at_inner(root.path(), path, interrupted, FailurePoint::AfterPointerRenameUnreadable), + Err(PublishError::Storage) + )); + let after = fs::read_dir(root.path().join("generations")) + .expect("generations after unreadable pointer") + .count(); + assert_eq!(after, before + 1, "an uncertain pointer must not delete the new generation"); + fs::remove_file(root.path().join(CURRENT)).expect("simulate accepted pointer loss"); + assert!( + matches!(current_directory(root.path()), Err(PublishError::Invalid)), + "an acknowledged publisher must never silently revert to bootstrap after pointer loss" + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn expired_generation_can_renew_and_next_operation_detects_corruption() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let expired_key = rcgen::KeyPair::generate().expect("expired key"); + let mut params = rcgen::CertificateParams::new(vec!["server.example".into(), "expired-required.example".into()]) + .expect("parameters"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let expired = params.self_signed(&expired_key).expect("expired certificate"); + let certificate = expired.pem(); + let key = expired_key.serialize_pem(); + let revision = Uuid::new_v4(); + let generations = root.path().join("generations"); + fs::create_dir(&generations).expect("generations"); + let directory = generations.join(revision.to_string()); + fs::create_dir(&directory).expect("generation"); + write_new(&directory.join(RUSTFS_TLS_CERT), certificate.as_bytes()).expect("expired cert"); + write_new(&directory.join(RUSTFS_TLS_KEY), key.as_bytes()).expect("expired key"); + let leaf = parse_leaf(expired.der().as_ref()).expect("expired metadata"); + let manifest = Manifest { + schema_version: SCHEMA.into(), + issuance_id: Uuid::new_v4(), + target_revision: revision, + certificate_fingerprint: leaf.fingerprint, + not_after_unix: leaf.not_after_unix, + payload_digest: payload_digest(certificate.as_bytes(), key.as_bytes()), + }; + write_new(&directory.join(MANIFEST), &serde_json::to_vec(&manifest).expect("manifest")).expect("manifest file"); + std::os::unix::fs::symlink(Path::new("generations").join(revision.to_string()), root.path().join(CURRENT)) + .expect("pointer"); + mark_initialized(root.path()).expect("initialized"); + let state = state_at(root.path(), path).expect("expired visible head remains readable"); + assert!(state.not_after_unix < 1_000_000_000); + assert!( + current_generation(root.path()) + .expect("expired generation") + .expect("present") + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .is_err(), + "reading an expired ledger must never bypass serving validity checks" + ); + assert!( + matches!( + publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))), + Err(PublishError::Invalid) + ), + "expired durable SAN coverage must survive renewal" + ); + let renewed = + rcgen::generate_simple_self_signed(vec!["server.example".into(), "expired-required.example".into()]) + .expect("renewal"); + let request = build_request(&state.target_revision, &renewed); + let retry = clone_request(&request); + let receipt = publish_at(root.path(), path, request).expect("renew expired generation"); + assert_ne!(receipt.target_revision, state.target_revision); + let generation = current_generation(root.path()).expect("published").expect("generation"); + assert_eq!( + state_at(root.path(), path).expect("first independent read").target_revision, + receipt.target_revision + ); + let directory = root + .path() + .join("generations") + .join(generation.manifest.target_revision.to_string()); + fs::write(directory.join(RUSTFS_TLS_KEY), "corrupted key").expect("corrupt after read"); + let snapshot = generation + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .expect("the pinned operation consumes its already verified material without a second file read"); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + assert!( + matches!(state_at(root.path(), path), Err(PublishError::Invalid)), + "the next operation must revalidate disk" + ); + assert!( + matches!(publish_at(root.path(), path, retry), Err(PublishError::Invalid)), + "idempotent retry must not bypass disk corruption" + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn invalid_candidate_never_changes_pointer() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("base path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let expected = state_at(root.path(), path).expect("state").target_revision; + assert!(matches!( + publish_at(root.path(), path, build_request(&expected, &pair("wrong.example"))), + Err(PublishError::Invalid) + )); + let valid = pair("server.example"); + let mut mismatch = build_request(&expected, &valid); + mismatch.private_key_pem = pair("server.example").signing_key.serialize_pem(); + assert!(matches!(publish_at(root.path(), path, mismatch), Err(PublishError::Invalid))); + let mut ca_params = rcgen::CertificateParams::new(vec!["server.example".into()]).expect("CA params"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca_cert = ca_params.self_signed(&ca_key).expect("CA certificate"); + let mut ca_candidate = build_request(&expected, &valid); + ca_candidate.certificate_pem = ca_cert.pem(); + ca_candidate.private_key_pem = ca_key.serialize_pem(); + assert!( + matches!(publish_at(root.path(), path, ca_candidate), Err(PublishError::Invalid)), + "a CA certificate must not become the serving leaf" + ); + let mut oversized = build_request(&expected, &valid); + oversized.certificate_pem = "x".repeat(MAX_CERT + 1); + assert!(matches!(publish_at(root.path(), path, oversized), Err(PublishError::Invalid))); + assert!(current_directory(root.path()).expect("pointer absent").is_none()); + assert_eq!(state_at(root.path(), path).expect("unchanged").target_revision, expected); + task.abort(); + }, + ) + .await; + } + + #[test] + fn corrupt_or_escaping_pointer_is_rejected() { + let root = tempfile::TempDir::new().expect("root"); + std::os::unix::fs::symlink("../outside", root.path().join(CURRENT)).expect("pointer"); + assert!(matches!(current_directory(root.path()), Err(PublishError::Invalid))); + fs::remove_file(root.path().join(CURRENT)).expect("remove pointer"); + fs::write(root.path().join(CURRENT), "generations/foo").expect("regular pointer"); + assert!(matches!(current_directory(root.path()), Err(PublishError::Invalid))); + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_is_observed_by_live_tls_handshakes() { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + let renewed = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (ENV_TLS_PUBLISH_DIR, root.path().to_str()), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (rustfs_config::ENV_TRUST_SYSTEM_CA, Some("0")), + (rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (rustfs_config::ENV_RUSTFS_TLS_PATH, Some("ignored-env-source")), + ], + async { + let path = base.path().to_str().expect("path"); + let initial = tls_material::load_tls_material(path).await.expect("initial snapshot"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let registration = tls_material::spawn_reload_loop( + path.into(), + holder.clone(), + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + assert_eq!( + tls_material::active_server_tls_path(), + Some(path.to_string()), + "registered CLI/listener source wins over environment" + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let address = listener.local_addr().expect("address"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (socket, _) = listener.accept().await.expect("TCP connection"); + let _tls = holder.get().accept(socket).await.expect("TLS handshake"); + } + }); + let mut roots = rustls::RootCertStore::empty(); + roots.add(original.cert.der().clone()).expect("bootstrap anchor"); + roots.add(renewed.cert.der().clone()).expect("renewal anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let connector = tokio_rustls::TlsConnector::from(std::sync::Arc::new(config)); + async fn handshake(connector: &tokio_rustls::TlsConnector, address: std::net::SocketAddr) -> String { + let socket = tokio::net::TcpStream::connect(address).await.expect("connect"); + let stream = connector + .connect(rustls::pki_types::ServerName::try_from("server.example").expect("SNI"), socket) + .await + .expect("verified handshake"); + hex_digest(stream.get_ref().1.peer_certificates().expect("peer chain")[0].as_ref()) + } + assert_eq!(handshake(&connector, address).await, hex_digest(original.cert.der().as_ref())); + let bootstrap = certificate_state(path.into()).await.expect("bootstrap state"); + let receipt = publish(path.into(), build_request(&bootstrap.target_revision, &renewed)) + .await + .expect("publish"); + let expected = hex_digest(renewed.cert.der().as_ref()); + assert_eq!(receipt.certificate_fingerprint, expected); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if certificate_state(path.into()) + .await + .expect("state") + .observed_certificate_fingerprint + == Some(expected.clone()) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("publication observed by serving group"); + assert_eq!(handshake(&connector, address).await, expected); + let mut expected_chain = vec![renewed.cert.der().clone()]; + assert_eq!(observed(path, &expected_chain), Some(expected.clone())); + expected_chain.push(original.cert.der().clone()); + assert!( + observed(path, &expected_chain).is_none(), + "matching leaf alone must not acknowledge a different serving chain" + ); + server.await.expect("TLS server"); + registration.abort(); + }, + ) + .await; + } +} diff --git a/rustfs/src/startup_runtime_sources.rs b/rustfs/src/startup_runtime_sources.rs index 30b35458f..73501a592 100644 --- a/rustfs/src/startup_runtime_sources.rs +++ b/rustfs/src/startup_runtime_sources.rs @@ -17,6 +17,7 @@ use crate::runtime_sources::current_outbound_tls_generation as runtime_current_o use crate::storage_api::startup::runtime_sources::{InstanceContext, set_global_rustfs_port}; use rustfs_kms::KmsServiceManager; use rustfs_obs::{GlobalError as ObservabilityError, OtelGuard}; +pub(crate) use rustfs_tls_runtime::load_global_outbound_tls_state as current_outbound_tls_state; use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration}; use std::sync::Arc; use tokio_util::sync::CancellationToken; diff --git a/rustfs/src/startup_tls_material.rs b/rustfs/src/startup_tls_material.rs index 9d770293e..32931c707 100644 --- a/rustfs/src/startup_tls_material.rs +++ b/rustfs/src/startup_tls_material.rs @@ -14,7 +14,9 @@ use crate::config::Config; use crate::startup_runtime_sources; +use std::collections::BTreeSet; use std::io::{Error, Result}; +use std::sync::Mutex; use tracing::{error, info}; const LOG_COMPONENT_MAIN: &str = "main"; @@ -23,6 +25,25 @@ const EVENT_TLS_OUTBOUND_INITIALIZED: &str = "tls_outbound_initialized"; const EVENT_TLS_OUTBOUND_INITIALIZATION_FAILED: &str = "tls_outbound_initialization_failed"; const TLS_STARTUP_GENERATION_CONSUMER: &str = "rustfs_server_startup"; +// Startup completes before listeners register; retain source names independently +// of the later listener snapshot, without pinning renewable certificate bytes. +static STARTUP_MATERIAL_FILES: Mutex)>> = Mutex::new(None); + +pub(crate) fn required_startup_material_files(tls_path: &str) -> BTreeSet { + STARTUP_MATERIAL_FILES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .filter(|(path, _)| path == tls_path.trim()) + .map(|(_, files)| files.clone()) + .unwrap_or_default() +} + +#[cfg(test)] +pub(crate) fn replace_startup_material_files(files: Option<(String, BTreeSet)>) -> Option<(String, BTreeSet)> { + std::mem::replace(&mut *STARTUP_MATERIAL_FILES.lock().unwrap_or_else(|error| error.into_inner()), files) +} + pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { crate::server::tls_material::validate_configured_oidc_extra_ca_cert() .await @@ -32,6 +53,8 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { match crate::server::tls_material::load_tls_material(tls_path).await { Ok(snapshot) => { publish_outbound_tls_material(&snapshot.outbound, Some(tls_path)).await; + *STARTUP_MATERIAL_FILES.lock().unwrap_or_else(|error| error.into_inner()) = + Some((tls_path.to_string(), snapshot.material_files)); } Err(err) => { error!( @@ -49,6 +72,11 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { if startup_runtime_sources::observability_metric_enabled() { startup_runtime_sources::init_tls_metrics(); } + } else { + STARTUP_MATERIAL_FILES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); } Ok(()) diff --git a/scripts/test_helm_templates.sh b/scripts/test_helm_templates.sh index 1b21e68cb..f273430b9 100755 --- a/scripts/test_helm_templates.sh +++ b/scripts/test_helm_templates.sh @@ -568,7 +568,7 @@ nondefault_statefulset=$(awk ' nondefault_local_host=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$nondefault_statefulset") nondefault_namespace=$(yq eval '.metadata.namespace' - <<<"$nondefault_statefulset") nondefault_endpoint_port=$(yq eval '.spec.template.spec.containers[0].ports[] | select(.name == "endpoint") | .containerPort' - <<<"$nondefault_statefulset") -mtls_liveness_command=$(yq eval '.spec.template.spec.containers[0].livenessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") +mtls_liveness_port=$(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$nondefault_statefulset") mtls_readiness_command=$(yq eval '.spec.template.spec.containers[0].readinessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") if [[ "$nondefault_local_host" != '$(RUSTFS_CHART_POD_NAME).tenant-rustfs-headless.object-storage.svc.cluster.corp' ]]; then echo "Combined generated topology produced an unexpected local endpoint host: $nondefault_local_host" >&2 @@ -578,9 +578,8 @@ if [[ "$nondefault_namespace" != "object-storage" || "$nondefault_endpoint_port" echo "Combined generated topology must retain its namespace and custom endpoint port" >&2 exit 1 fi -if ! grep -Fq 'https://127.0.0.1:9443/health' <<<"$mtls_liveness_command" || - grep -Fq '/health/ready' <<<"$mtls_liveness_command"; then - echo "mTLS liveness must use the process health endpoint, not readiness" >&2 +if [[ "$mtls_liveness_port" != 9443 ]]; then + echo "mTLS liveness must check the listener independently of certificate validity" >&2 exit 1 fi if ! grep -Fq 'https://127.0.0.1:9443/health/ready' <<<"$mtls_readiness_command"; then diff --git a/scripts/test_helm_tls_reload.sh b/scripts/test_helm_tls_reload.sh new file mode 100644 index 000000000..e8ad68d86 --- /dev/null +++ b/scripts/test_helm_tls_reload.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CHART_DIR="$ROOT_DIR/helm/rustfs" + +for distributed in false true; do + rendered=$(helm template tls-reload "$CHART_DIR" \ + --set secret.allowInsecureDefaults=true \ + --set mode.distributed.enabled="$distributed" \ + --set mode.standalone.enabled="$([[ "$distributed" == false ]] && echo true || echo false)" \ + --set tls.enabled=true --set tls.existingSecret=issued-tls) + workload=$(yq eval 'select(.kind == "Deployment" or .kind == "StatefulSet")' - <<<"$rendered") + mount=$(yq eval '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "tls-material")' - <<<"$workload") + [[ $(yq eval '.mountPath' - <<<"$mount") == /opt/tls ]] + [[ $(yq eval '.readOnly' - <<<"$mount") == true ]] + [[ $(yq eval 'has("subPath")' - <<<"$mount") == false ]] + keys=$(yq eval '.spec.template.spec.volumes[] | select(.name == "tls-material") | .projected.sources[].secret.items[].key' - <<<"$workload") + [[ "$keys" == $'tls.crt\ntls.key' ]] + [[ $(yq eval '.spec.template.spec.containers[0].readinessProbe.httpGet.scheme' - <<<"$workload") == HTTPS ]] + [[ $(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$workload") == 9000 ]] + config=$(yq eval 'select(.kind == "ConfigMap") | .data' - <<<"$rendered") + [[ $(yq eval '.RUSTFS_TLS_RELOAD_ENABLE' - <<<"$config") == 1 ]] + [[ $(yq eval '.RUSTFS_TLS_KEYLOG' - <<<"$config") == 0 ]] + if [[ "$distributed" == true ]]; then + [[ $(yq eval '.RUSTFS_VOLUMES' - <<<"$config") == https://* ]] + fi + + mtls=$(helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set mode.distributed.enabled="$distributed" \ + --set mode.standalone.enabled="$([[ "$distributed" == false ]] && echo true || echo false)" \ + --set mtls.enabled=true) + workload=$(yq eval 'select(.kind == "Deployment" or .kind == "StatefulSet")' - <<<"$mtls") + [[ $(yq eval '.spec.template.spec.volumes[] | select(.name == "tls-material") | .projected.sources | length' - <<<"$workload") == 2 ]] + [[ $(yq eval '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "tls-material") | has("subPath")' - <<<"$workload") == false ]] + [[ $(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$workload") == 9000 ]] +done + +if helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set tls.enabled=true --set tls.existingSecret=issued-tls --set mtls.enabled=true >/dev/null 2>&1; then + echo "Public TLS must not silently enable global client authentication" >&2 + exit 1 +fi +if helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set tls.enabled=true >/dev/null 2>&1; then + echo "Native TLS requires an explicitly configured serving Secret" >&2 + exit 1 +fi +echo "TLS Secret projection, rotation settings, topology and probes passed"