diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index eb30d58e6..5aaf9163f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -332,6 +332,8 @@ The binary (`main.rs`) boots in this order: - **"Where is replication configured?"** `admin/handlers/replication.rs` and `admin/handlers/site_replication.rs` for API, + `rustfs/src/site_replication/` for the site-replication service subsystem + (state, peer transport, retry queue, repair, hooks), `ecstore/src/bucket/replication/` for engine - **"Where do I add a new admin endpoint?"** diff --git a/docs/architecture/ecstore-config-consumer-inventory.md b/docs/architecture/ecstore-config-consumer-inventory.md index 96789d516..4a0a91c8a 100644 --- a/docs/architecture/ecstore-config-consumer-inventory.md +++ b/docs/architecture/ecstore-config-consumer-inventory.md @@ -123,7 +123,7 @@ behind narrower contracts. | Files | Current usage | |---|---| | `rustfs/src/admin/handlers/kms_dynamic.rs` | Uses generic `read_config` and `save_config` for dynamic KMS config objects. | -| `rustfs/src/admin/handlers/site_replication.rs` | Uses generic `read_config`, `save_config`, and `delete_config` for site-replication state objects. | +| `rustfs/src/site_replication/state.rs` | Uses generic `read_config`, `save_config`, and `delete_config` (via the root storage facade) for site-replication state objects. | | `rustfs/src/admin/service/site_replication.rs` | Uses generic `read_config` and `save_config` for site-replication state normalization. | | `rustfs/src/server/module_switch.rs` | Uses generic `read_config` and `save_config` for module-switch config objects. | | `crates/iam/src/store/object.rs` | Uses generic `read_config_no_lock`, `read_config_with_metadata`, `save_config`, `save_config_with_opts`, and `delete_config` helper variants for IAM object-store persistence paths. | diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index cbb2db326..66726639d 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -110,11 +110,11 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object/get.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | | `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. | | `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. | -| `SITE_REPLICATION_PEER_CLIENT` | `rustfs/src/admin/handlers/site_replication.rs` | Process-global owner-local cache | Site-replication peer client cache stays private to site-replication handlers. The state RMW transaction holds no process-local mutex — see `rustfs/src/admin/site_replication_state.rs`. | +| `SITE_REPLICATION_PEER_CLIENT` | `rustfs/src/site_replication/transport.rs` | Process-global owner-local cache | Site-replication peer client cache stays private to the site-replication transport module. The state RMW transaction holds no process-local mutex — see `rustfs/src/site_replication/state_lock.rs`. | | `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED`, `PERSISTED_NOTIFY_MODULE_ENABLED`, `PERSISTED_AUDIT_MODULE_ENABLED`, `PERSISTED_MODULE_SWITCH_CONFIGURED` | `rustfs/src/server/audit.rs`, `rustfs/src/server/event.rs`, `rustfs/src/server/module_switch.rs` | Process-global owner-local toggles | Audit/notify module snapshots stay private to the server module switch owners. | | `DELETE_TAIL_TOTAL`, `DELETE_CLEANUP_TOTAL`, `DELETE_REPLICATION_TOTAL`, `DELETE_NOTIFY_TOTAL` | `rustfs/src/delete_tail_activity.rs` | Process-global owner-local counters | Delete-tail activity counters stay private behind delete-tail activity helpers. | | `EMBEDDED_SERVER_STARTED` | `rustfs/src/startup_lifecycle.rs` | Process-global owner-local guard | Embedded startup single-start protection stays private to startup lifecycle. | -| `TEST_OUTBOUND_TLS_GENERATION` | `rustfs/src/admin/runtime_sources.rs` | Test or fixture state | Outbound TLS generation test hook state stays private to admin runtime-source tests. | +| `TEST_OUTBOUND_TLS_GENERATION` | `rustfs/src/site_replication/mod.rs` | Test or fixture state | Outbound TLS generation test hook state stays private to site-replication transport tests. | | `TEST_REMAINING_FAILURES` | `rustfs/src/startup_iam.rs` | Test or fixture state | IAM startup retry injection state stays private to debug/test startup code. | | `CAPACITY_DIRTY_SCOPE_ENV`, `CAPACITY_DIRTY_SCOPE_INIT`, `GLOBAL_ENV`, function-local `INIT` | `rustfs/src/app/*_test.rs` | Test or fixture state | App integration test fixture state stays private to the owning test modules. | diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 3881b22eb..476777a0e 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -15,14 +15,9 @@ use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ - current_deployment_id, current_endpoints_handle, current_federated_identity_service, current_iam_handle, - current_object_store_handle, current_outbound_tls_generation, current_outbound_tls_state, current_region, - current_replication_pool_handle, current_replication_stats_handle, current_runtime_port, current_server_config, - current_token_signing_key, object_store_from_req, -}; -use crate::admin::site_replication_identity::{ - canonical_endpoint, deployment_id_for_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled, - normalize_peer_map_by_identity_with, same_identity_endpoint, site_identity_key, + current_deployment_id, current_federated_identity_service, current_iam_handle, current_object_store_handle, current_region, + current_replication_pool_handle, current_replication_stats_handle, current_server_config, current_token_signing_key, + object_store_from_req, }; use crate::admin::storage_api::bucket::metadata::{ BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, @@ -32,97 +27,74 @@ use crate::admin::storage_api::bucket::metadata_sys; use crate::admin::storage_api::bucket::quota::BucketQuota; use crate::admin::storage_api::bucket::replication; use crate::admin::storage_api::bucket::replication::{ - OperatorRuleContract, assign_site_replication_rule_priorities, is_site_replication_role, merge_incoming_replication_config, - replication_target_arn_deployment_id, site_replication_rule_deployment_id, + OperatorRuleContract, assign_site_replication_rule_priorities, merge_incoming_replication_config, + replication_target_arn_deployment_id, }; -use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; -use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; +use crate::admin::storage_api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets}; use crate::admin::storage_api::bucket::utils::{deserialize, serialize}; use crate::admin::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; -use crate::admin::storage_api::config::read_admin_config; -#[cfg(test)] -use crate::admin::storage_api::config::save_admin_config; use crate::admin::storage_api::contract::bucket::{ BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, }; use crate::admin::storage_api::error::{Error as StorageError, is_err_bucket_not_found}; -use crate::admin::storage_api::runtime::ECStore; -use crate::admin::utils::{empty_response, encode_compatible_admin_payload, json_response, read_compatible_admin_body}; -use crate::auth::constant_time_eq; -use crate::config::get_config_snapshot; +use crate::admin::utils::{empty_response, json_response, read_compatible_admin_body}; use crate::error::ApiError; use crate::server::ADMIN_PREFIX; -use crate::storage::storage_api::{ - delete_config_no_lock, lock_bucket_targets_metadata, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, - with_config_object_write_lock, +use crate::site_replication::identity::{ + canonical_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled, same_identity_endpoint, site_identity_key, }; -use base64_simd::STANDARD as BASE64_STANDARD; +use crate::storage::storage_api::{lock_bucket_targets_metadata, with_config_object_write_lock}; use base64_simd::URL_SAFE_NO_PAD; use futures::StreamExt; -use hmac::{Hmac, Mac}; -use http::header::{CONTENT_TYPE, HOST}; -use http::{HeaderMap, Uri}; +use http::Uri; use hyper::{Method, StatusCode}; use matchit::Params; -use rustfs_config::{ - DEFAULT_CONSOLE_ADDRESS, DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH, - MAX_ADMIN_REQUEST_BODY_SIZE, -}; +use rustfs_config::{DEFAULT_DELIMITER, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_iam::error::is_err_no_such_service_account; use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM; use rustfs_iam::store::object::ObjectStore; -use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type, user_type_from_sr_wire}; +use rustfs_iam::store::user_type_from_sr_wire; use rustfs_iam::sys::{ IamSys, NewServiceAccountOpts, SITE_REPLICATOR_SERVICE_ACCOUNT, UpdateServiceAccountOpts, get_claims_from_token_with_secret, }; use rustfs_madmin::{ - AddOrUpdateUserReq, BucketBandwidth, GroupAddRemove, GroupStatus, IDPSettings, InProgressMetric, InQueueMetric, - LDAPConfigSettings, LDAPSettings, OpenIDProviderSettings, PeerInfo, PeerSite, QStat, ReplProxyMetric, ReplicateAddStatus, - ReplicateEditStatus, ReplicateRemoveStatus, ResyncBucketStatus, SITE_REPL_API_VERSION, SR_IAM_ITEM_STS_ACC, - SR_IAM_ITEM_STS_ACC_LEGACY, SRBucketInfo, SRBucketMeta, SRBucketStatsSummary, SRGroupInfo, SRGroupStatsSummary, SRIAMItem, - SRIAMPolicy, SRIAMUser, SRILMExpiryStatsSummary, SRInfo, SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, - SRPendingOperation, SRPolicyMapping, SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRRetryStats, SRSTSCredential, - SRSessionPolicy, SRSiteSummary, SRStateEditReq, SRStateInfo, SRStatusInfo, SRSvcAccChange, SRSvcAccCreate, - SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, + BucketBandwidth, GroupStatus, IDPSettings, InProgressMetric, InQueueMetric, LDAPConfigSettings, LDAPSettings, + OpenIDProviderSettings, PeerInfo, PeerSite, QStat, ReplProxyMetric, ReplicateAddStatus, ReplicateEditStatus, + ReplicateRemoveStatus, ResyncBucketStatus, SITE_REPL_API_VERSION, SR_IAM_ITEM_STS_ACC, SR_IAM_ITEM_STS_ACC_LEGACY, + SRBucketMeta, SRBucketStatsSummary, SRGroupInfo, SRGroupStatsSummary, SRIAMItem, SRIAMUser, SRILMExpiryStatsSummary, SRInfo, + SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, SRPendingOperation, SRPolicyMapping, SRPolicyStatsSummary, + SRRemoveReq, SRResyncOpStatus, SRSTSCredential, SRSessionPolicy, SRSiteSummary, SRStateEditReq, SRStateInfo, SRStatusInfo, + SRSvcAccChange, SRSvcAccCreate, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, }; use rustfs_policy::policy::{ Policy, action::{Action, AdminAction}, }; -use rustfs_signer::constants::UNSIGNED_PAYLOAD; -use rustfs_signer::sign_v4; -use rustfs_tls_runtime::GlobalPublishedOutboundTlsState; -use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url}; -use rustfs_utils::http::get_source_scheme; -use rustls_pki_types::pem::PemObject; use s3s::dto::{ - BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, - Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus, - ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria, VersioningConfiguration, + DeleteMarkerReplicationStatus, DeleteReplicationStatus, ExistingObjectReplicationStatus, ReplicaModificationsStatus, + ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, }; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::Deserialize; use serde::Serialize; -use serde::de::{DeserializeOwned, IgnoredAny}; +use serde::de::DeserializeOwned; use serde_json::Value; -use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::net::{IpAddr, SocketAddr}; -use std::sync::{Arc, LazyLock, Mutex as StdMutex}; +use std::sync::{LazyLock, Mutex as StdMutex}; use std::time::Duration; use time::OffsetDateTime; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::Mutex; use tracing::{info, warn}; -use url::{Url, form_urlencoded}; +use url::form_urlencoded; use uuid::Uuid; -const LOG_COMPONENT_ADMIN: &str = "admin"; -const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication"; -const EVENT_ADMIN_SITE_REPLICATION_STATE: &str = "admin_site_replication_state"; +// The site-replication service subsystem (state, peer transport, retry queue, +// repair state machine, broadcast hooks) lives in `crate::site_replication` +// (backlog#1840); re-export it so existing `admin::handlers::site_replication` +// paths keep resolving while this file keeps only the HTTP handlers. +pub(crate) use crate::site_replication::*; + const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2; -use crate::admin::site_replication_state::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock}; -const SITE_REPLICATION_REPAIR_STATE_PATH: &str = "config/site-replication/repair-state.json"; -const SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH: &str = "config/site-replication/repair-execution.lock"; // Serializes peer-join admission (staleness check -> IAM upsert -> state // commit) across every node of this site; see admit_peer_join. Never an // actual object — only a namespace-lock key, like the repair execution lock. @@ -139,50 +111,22 @@ const SITE_REPL_RESYNC_CANCEL: &str = "cancel"; const SITE_REPL_RESYNC_STATUS: &str = "status"; const SITE_REPL_RESYNC_DEFAULT_PAGE_SIZE: usize = 100; const SITE_REPL_RESYNC_MAX_PAGE_SIZE: usize = 1000; -const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); -const SITE_REPLICATION_PEER_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); /// Bound on waiting for the lifecycle lock (below). 3x the peer request /// timeout: outlives one full peer round of a healthy concurrent lifecycle /// operation, while converting a holder wedged on unreachable peers into a /// retryable 503 for the waiter instead of an unbounded hang. const SITE_REPLICATION_LIFECYCLE_LOCK_TIMEOUT: Duration = Duration::from_secs(30); -const SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT: usize = 256; const SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT: usize = 32; -const MAX_PEER_CA_CERT_PEM_SIZE: usize = 256 * 1024; -const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET"; -const SITE_REPLICATION_RETRY_QUEUE_LIMIT: usize = 256; -const SITE_REPLICATION_RETRY_FAILED_AFTER: u32 = 3; -const SITE_REPLICATION_REPAIR_OPERATION_LIMIT: usize = 32; -const SITE_REPLICATION_REPAIR_IAM_FAMILY: &str = "iam"; -const SITE_REPLICATION_REPAIR_BUCKET_FAMILY: &str = "bucket"; -const SITE_REPLICATION_REPAIR_BUCKET_METADATA_FAMILY: &str = "bucket-metadata"; -const SITE_REPLICATION_REPAIR_REPLICATION_FAMILY: &str = "replication"; -const SITE_REPLICATION_PEER_BUCKET_OPS_PATH: &str = "/rustfs/admin/v3/site-replication/peer/bucket-ops"; -const SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING: &str = "make-with-versioning"; -const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "configure-replication"; const IDENTITY_LDAP_SUB_SYS: &str = "identity_ldap"; const LEGACY_LDAP_SUB_SYS: &str = "ldapserverconfig"; const SITE_REPLICATION_PEER_JOIN_PATH: &str = "/rustfs/admin/v3/site-replication/peer/join"; -const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit"; const SITE_REPLICATION_PEER_EDIT_CAPABILITY_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=endpoint-target-refresh"; const SITE_REPLICATION_PEER_TLS_CAPABILITY_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=peer-tls-settings"; -const SITE_REPLICATION_PEER_DERIVED_RULE_CONTRACT_CAPABILITY_PATH: &str = - "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=derived-rule-contract"; const SITE_REPLICATION_PEER_EDIT_REFRESH_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit?refresh-targets=true"; -/// Peer-edit fencing token, carried as query parameters so a peer that predates -/// the fence simply ignores them (unknown query keys are dropped) and keeps the -/// previous last-writer-wins behaviour. -const SITE_REPLICATION_EDIT_ORIGIN_QUERY: &str = "editOrigin"; -const SITE_REPLICATION_EDIT_GENERATION_QUERY: &str = "editGeneration"; -const SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH: &str = "internal:endpoint-target-refresh"; const SITE_REPLICATION_PEER_REMOVE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove"; const SITE_REPLICATION_DEVNULL_PATH: &str = "/rustfs/admin/v3/site-replication/devnull"; -const RUSTFS_ADMIN_V3_PREFIX: &str = "/rustfs/admin/v3"; -const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3"; -const MINIO_SITE_REPLICATION_PEER_JOIN_PATH: &str = "/minio/admin/v3/site-replication/peer/join"; - fn site_replicator_service_account_policy() -> S3Result { Policy::parse_config( br#"{ @@ -242,144 +186,13 @@ fn site_replicator_service_account_policy() -> S3Result { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}"))) } -#[derive(Clone)] -enum SiteReplicationPeerClientCacheEntry { - Ready(reqwest::Client), - Failed(String), -} - -#[derive(Clone)] -struct SiteReplicationPeerClientCache { - generation: u64, - entry: SiteReplicationPeerClientCacheEntry, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PeerConnection { - endpoint: Url, - skip_tls_verify: bool, - ca_cert_pem: String, -} - -#[derive(Deserialize, Default)] -struct PeerTlsFieldPresence { - #[serde(rename = "skipTlsVerify")] - skip_tls_verify: Option, - #[serde(rename = "caCertPem")] - ca_cert_pem: Option, -} - -impl PeerTlsFieldPresence { - fn has_skip_tls_verify(&self) -> bool { - self.skip_tls_verify.is_some() - } - - fn has_ca_cert_pem(&self) -> bool { - self.ca_cert_pem.is_some() - } -} - -#[derive(Clone)] -struct PeerDnsResolver { - allow_loopback: bool, - #[cfg(test)] - overrides: Option>>>, -} - -impl PeerDnsResolver { - fn new(allow_loopback: bool) -> Self { - Self { - allow_loopback, - #[cfg(test)] - overrides: None, - } - } - - #[cfg(test)] - fn with_overrides(allow_loopback: bool, overrides: HashMap>) -> Self { - Self { - allow_loopback, - overrides: Some(Arc::new(overrides)), - } - } -} - -impl reqwest::dns::Resolve for PeerDnsResolver { - fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { - let host = name.as_str().to_string(); - let allow_loopback = self.allow_loopback; - #[cfg(test)] - let overrides = self.overrides.clone(); - Box::pin(async move { - #[cfg(test)] - let overridden = overrides.as_ref().and_then(|entries| entries.get(&host)).cloned(); - #[cfg(not(test))] - let overridden: Option> = None; - - let ips = if let Some(ips) = overridden { - ips - } else { - tokio::net::lookup_host((host.as_str(), 0)) - .await? - .map(|addr| addr.ip()) - .collect() - }; - let addrs = ips - .into_iter() - .filter(|ip| resolved_peer_ip_allowed(&host, *ip, allow_loopback)) - .map(|ip| SocketAddr::new(ip, 0)) - .collect::>(); - if addrs.is_empty() { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - format!("site replication DNS resolution for `{host}` returned no allowed addresses"), - ) - .into()); - } - Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) - }) - } -} - -impl PeerConnection { - fn new(endpoint: &str, skip_tls_verify: bool, ca_cert_pem: &str) -> S3Result { - validate_peer_connection_inner(endpoint, skip_tls_verify, ca_cert_pem, loopback_replication_targets_allowed()) - } - - fn endpoint(&self) -> &str { - self.endpoint.as_str().trim_end_matches('/') - } - - fn uses_default_tls(&self) -> bool { - !self.skip_tls_verify && self.ca_cert_pem.is_empty() - } -} - -impl TryFrom<&PeerInfo> for PeerConnection { - type Error = S3Error; - - fn try_from(peer: &PeerInfo) -> Result { - Self::new(&peer.endpoint, peer.skip_tls_verify, &peer.ca_cert_pem) - } -} - -impl TryFrom<&PeerSite> for PeerConnection { - type Error = S3Error; - - fn try_from(site: &PeerSite) -> Result { - Self::new(&site.endpoint, site.skip_tls_verify, &site.ca_cert_pem) - } -} - -static SITE_REPLICATION_PEER_CLIENT: LazyLock>> = LazyLock::new(|| Mutex::new(None)); // Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata. // "state" is the distributed state-object lock in -// crate::admin::site_replication_state, entered through +// crate::site_replication::state_lock, entered through // update_site_replication_state (P1-15). There is no process-local state // mutex any more: it could not order two nodes of one site, and the call // sites that needed ordering carry a generation fence instead. static SITE_REPLICATION_LIFECYCLE_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); -static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock> = LazyLock::new(|| RwLock::new(())); static SITE_REPLICATION_ADD_BOOTSTRAP: LazyLock>> = LazyLock::new(|| StdMutex::new(None)); @@ -461,212 +274,6 @@ fn bootstrap_peer_bucket_operation_allowed(bucket: &str, operation: &str, bootst }) } -fn site_replication_peer_client_cache_hit( - cache: &Option, - generation: u64, -) -> Option> { - let cached = cache.as_ref()?; - if cached.generation != generation { - return None; - } - Some(match &cached.entry { - SiteReplicationPeerClientCacheEntry::Ready(client) => Ok(client.clone()), - SiteReplicationPeerClientCacheEntry::Failed(err) => Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("initialize site replication peer client failed: {err}"), - )), - }) -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -struct SiteReplicationState { - name: String, - service_account_access_key: String, - #[serde(default, skip_serializing)] - service_account_secret_key: String, - service_account_parent: String, - peers: BTreeMap, - updated_at: Option, - resync_status: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pending_rotation: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pending_remove: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pending_endpoint_refresh: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - retry_queue: Vec, - #[serde(default)] - sync_state_initialized: bool, - /// Fencing token for peer-edit delivery, allocated inside the state - /// transaction (the distributed state-object lock). Two nodes of THIS - /// site that accept admin edits concurrently therefore get strictly - /// ordered generations, and a delivery that stalls can be recognised as - /// stale by the receiving site. - #[serde(default)] - edit_generation: u64, - /// Per-origin high-water mark of the peer edits already applied here, - /// keyed by the origin site's deployment id. A delivery whose generation - /// is not above the mark arrived out of order and must not overwrite the - /// newer edit that already landed. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - applied_edit_generations: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairState { - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - operations: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairOperation { - operation_id: String, - preflight_token: String, - plan_token: String, - status: String, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - sites: BTreeMap, - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - created_at: Option, - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - updated_at: Option, - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - completed_at: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairSiteStatus { - deployment_id: String, - name: String, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - families: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairFamilyStatus { - planned: usize, - succeeded: usize, - failed: usize, - #[serde(default)] - retry_events: usize, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - tasks: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - errors: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairTaskStatus { - task_id: String, - status: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - error: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct SiteReplicationRepairRequest { - mode: SiteReplicationRepairMode, - #[serde(default)] - preflight_token: Option, - #[serde(default)] - operation_id: Option, -} - -#[derive(Debug, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -enum SiteReplicationRepairMode { - DryRun, - Execute, -} - -struct SiteReplicationRepairExecutionRequest { - local_peer: PeerInfo, - preflight_token: String, - operation_id: String, - signing_key: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairPreflight { - mode: &'static str, - status: &'static str, - preflight_token: String, - retry_events: usize, - sites: BTreeMap, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairOperationResponse { - mode: &'static str, - operation_id: String, - status: String, - sites: BTreeMap, - #[serde(with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - created_at: Option, - #[serde(with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - updated_at: Option, - #[serde(with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - completed_at: Option, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairSiteResponse { - deployment_id: String, - name: String, - families: BTreeMap, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct SiteReplicationRepairFamilyResponse { - planned: usize, - succeeded: usize, - failed: usize, - retry_events: usize, - tasks: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - errors: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -struct SiteReplicationRetryEvent { - id: String, - peer_deployment_id: String, - peer_endpoint: String, - path: String, - retry_count: u32, - failed: bool, - last_error: String, - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - updated_at: Option, - /// Peer-edit generation whose delivery failed, when the failing send - /// carried one. Settling a *later* success for the same (peer, path) must - /// not erase a failure recorded for a NEWER generation — see - /// [`settle_site_replication_retry_events`]. - #[serde(default, skip_serializing_if = "Option::is_none")] - edit_generation: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -struct PendingEndpointRefresh { - id: String, - peer: PeerInfo, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - remote_peers: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] - acked_deployment_ids: BTreeSet, -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct EndpointRefreshRequest { @@ -674,43 +281,6 @@ struct EndpointRefreshRequest { peer: PeerInfo, } -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -struct PendingRotation { - id: String, - access_key: String, - parent: String, - new_secret_key: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - secret_candidates: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - peers: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] - acked_deployment_ids: BTreeSet, - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - updated_at: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -struct PendingRemove { - id: String, - req: SRRemoveReq, - service_account_access_key: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - secret_candidates: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - original_peers: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] - acked_deployment_ids: BTreeSet, - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] - updated_at: Option, -} - -struct SiteReplicationRuntime { - state: SiteReplicationState, - local_peer: PeerInfo, - service_account_secret_key: String, -} - #[derive(Debug, Clone)] struct SiteReplicationAddPreflightInfo { name: String, @@ -723,14 +293,6 @@ struct SiteReplicationAddPreflightInfo { idp_settings: serde_json::Value, } -#[derive(Debug, Default)] -struct SiteReplicationBootstrapPlan { - iam_items: Vec, - bucket_make_ops: Vec, - bucket_items: Vec, - bucket_configure_ops: Vec, -} - #[derive(Debug, Clone, Serialize, Deserialize, Default)] struct SRPeerJoinResponse { peer: PeerInfo, @@ -817,12 +379,6 @@ struct SiteNetPerfNodeResult { error: String, } -impl SiteReplicationState { - fn enabled(&self) -> bool { - self.peers.len() > 1 - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] enum SREntityType { #[default] @@ -1084,489 +640,6 @@ fn parse_public_peer_edit(body: &[u8]) -> S3Result<(PeerInfo, PeerTlsFieldPresen Ok((parse_site_replication_json(body)?, parse_site_replication_json(body)?)) } -fn parse_site_replication_state(data: &[u8]) -> S3Result { - let mut state: SiteReplicationState = serde_json::from_slice(data) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}")))?; - state.peers = normalize_peer_map_by_identity(state.peers); - // A peer-edit high-water mark only fences a CURRENT peer. A site that - // leaves drops below two peers, which clears its own state object and - // restarts its generation counter — a mark left over from the previous - // membership must not reject the edits it sends after it rejoins. This - // pruning covers departures THIS site observed; an origin removed - // unilaterally elsewhere stays in this peer map with its mark, and the - // wall-clock floor in `next_peer_edit_generation` is what lifts its - // restarted counter over that mark. Dropping departed origins on load - // also keeps the map bounded. - state - .applied_edit_generations - .retain(|origin, _| state.peers.contains_key(origin)); - if !state.sync_state_initialized { - if state.enabled() { - mark_unknown_peer_sync_enabled(&mut state.peers); - } - state.sync_state_initialized = true; - } - Ok(state) -} - -async fn load_site_replication_state() -> S3Result { - let Some(store) = current_object_store_handle() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - match read_admin_config(store, SITE_REPLICATION_STATE_PATH).await { - Ok(data) => parse_site_replication_state(&data), - Err(StorageError::ConfigNotFound) => Ok(SiteReplicationState::default()), - Err(err) => Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to load site replication state: {err}"), - )), - } -} - -/// Whether this deployment participates in site replication (two or more -/// peers in the persisted state). Read by the S3 interface layer to gate -/// replication-config edits (MinIO `ErrReplicationDenyEditError` semantics, -/// issue #1948); a state-read failure propagates so the gate fails closed. -pub(crate) async fn site_replication_enabled() -> S3Result { - Ok(load_site_replication_state().await?.enabled()) -} - -/// Deployment ids of the remote peers the reconciler derives a -/// `site-repl-` rule for on every bucket (the same peer filter as -/// `build_site_replication_config`); empty when site replication is not -/// enabled. Read by the bucket usecase so an S3 replication-config edit keeps -/// exactly the reconciler-owned rules (issue #1948); a state-read failure -/// propagates so the edit fails closed. -pub(crate) async fn site_replication_edit_context() -> S3Result<(HashSet, OperatorRuleContract)> { - let Some(runtime) = runtime_site_replication_targets().await? else { - // Enabled without a service account is a state this site cannot - // broadcast from either; the peers are still the reconciler's. - let state = load_site_replication_state().await?; - if !state.enabled() { - return Ok((HashSet::new(), OperatorRuleContract::Derived)); - } - let peers = remote_peer_deployment_ids(&state, ¤t_local_runtime_peer(&state)); - return Ok((peers, OperatorRuleContract::Legacy)); - }; - let peers = remote_peer_deployment_ids(&runtime.state, &runtime.local_peer); - let contract = site_replication_operator_rule_contract(&runtime).await; - Ok((peers, contract)) -} - -/// Whether every remote peer merges replication configs under the derived -/// contract, probed through the peer capability endpoint. A peer that does -/// not (or cannot be asked) pins the cluster to [`OperatorRuleContract::Legacy`] -/// for this edit: consistency across sites wins over keeping the operator's -/// priority values, and the legacy merge keeps their order anyway. -async fn site_replication_operator_rule_contract(runtime: &SiteReplicationRuntime) -> OperatorRuleContract { - let remote_peers: Vec<&PeerInfo> = runtime - .state - .peers - .values() - .filter(|peer| { - peer.deployment_id != runtime.local_peer.deployment_id - && !same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) - }) - .collect(); - let probes = futures::future::join_all(remote_peers.iter().map(|peer| async move { - let transport = PeerTransport::for_runtime_peer(peer).await?; - let (status, body) = send_peer_admin_request_raw_with_client( - &transport.client, - &transport.connection, - SITE_REPLICATION_PEER_DERIVED_RULE_CONTRACT_CAPABILITY_PATH, - &runtime.state.service_account_access_key, - &runtime.service_account_secret_key, - &(), - ) - .await?; - peer_capability_response_supported(peer, status, &body) - })) - .await; - operator_rule_contract_from_probes(remote_peers.into_iter().zip(probes)) -} - -fn operator_rule_contract_from_probes<'a>( - probes: impl IntoIterator)>, -) -> OperatorRuleContract { - for (peer, probe) in probes { - match probe { - Ok(true) => {} - Ok(false) => return OperatorRuleContract::Legacy, - Err(err) => { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - result = "derived_rule_contract_probe_failed", - peer = %peer.endpoint, - error = %err, - "admin site replication state" - ); - return OperatorRuleContract::Legacy; - } - } - } - OperatorRuleContract::Derived -} - -fn remote_peer_deployment_ids(state: &SiteReplicationState, local_peer: &PeerInfo) -> HashSet { - state - .peers - .values() - .filter(|peer| { - peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) - }) - .map(|peer| peer.deployment_id.clone()) - .collect() -} - -/// Deployment ids of every site in the cluster, this one included: the set -/// a peer's derived rules can name (its rule towards this site carries this -/// site's id). Empty when site replication is not enabled. -async fn site_replication_deployment_ids() -> S3Result> { - let state = load_site_replication_state().await?; - if !state.enabled() { - return Ok(HashSet::new()); - } - Ok(state.peers.values().map(|peer| peer.deployment_id.clone()).collect()) -} - -async fn load_site_replication_state_no_lock(store: Arc) -> S3Result { - match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await { - Ok(data) => parse_site_replication_state(&data), - Err(StorageError::ConfigNotFound) => Ok(SiteReplicationState::default()), - Err(err) => Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to load site replication state: {err}"), - )), - } -} - -/// Persist-or-clear under an already-held state object lock. Normalizes the -/// peer map exactly once (the historical persist path normalized twice with -/// two full clones — P2-22). -async fn persist_site_replication_state_no_lock(store: Arc, mut state: SiteReplicationState) -> S3Result<()> { - state.peers = normalize_peer_map_by_identity(state.peers); - if state.peers.len() <= 1 && state.pending_rotation.is_none() && state.pending_remove.is_none() { - match delete_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await { - Ok(()) | Err(StorageError::ConfigNotFound) => Ok(()), - Err(err) => Err(S3Error::with_message(S3ErrorCode::InternalError, format!("clear state failed: {err}"))), - } - } else { - let data = serde_json::to_vec(&state) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize state failed: {e}")))?; - save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, data) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save state failed: {e}"))) - } -} - -/// What a state transaction closure decided to do with the state it was -/// handed. `Unchanged` skips the write entirely: the ack markers and the -/// pending-clearing paths run on every retry and mostly find their pending id -/// already gone, and the retry queue shares this object — rewriting it byte -/// for byte only makes those misses contend with the writers that do have -/// something to say. -enum StateCommit { - Changed(T), - Unchanged(T), -} - -/// The site-replication state RMW transaction: load, mutate, persist — all -/// under the distributed state-object write lock (see -/// crate::admin::site_replication_state). No peer network calls and no other -/// config locks inside `update`; anything that has to talk to a peer belongs -/// between two transactions, with the precondition re-checked inside the -/// second one. -async fn update_site_replication_state(update: F) -> S3Result -where - T: Send + 'static, - F: FnOnce(&mut SiteReplicationState) -> S3Result + Send + 'static, -{ - update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await -} - -/// [`update_site_replication_state`] for closures that may find nothing to -/// do — see [`StateCommit`]. -async fn update_site_replication_state_when_changed(update: F) -> S3Result -where - T: Send + 'static, - F: FnOnce(&mut SiteReplicationState) -> S3Result> + Send + 'static, -{ - with_site_replication_state_lock(move || async move { - let store = current_object_store_handle() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - let mut state = load_site_replication_state_no_lock(store.clone()).await?; - match update(&mut state)? { - StateCommit::Changed(result) => { - persist_site_replication_state_no_lock(store, state).await?; - Ok(result) - } - StateCommit::Unchanged(result) => Ok(result), - } - }) - .await -} - -async fn load_site_replication_repair_state_from_store(store: Arc) -> S3Result { - match read_config_no_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH).await { - Ok(data) => serde_json::from_slice(&data).map_err(|e| { - S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication repair state: {e}")) - }), - Err(StorageError::ConfigNotFound) => Ok(SiteReplicationRepairState::default()), - Err(err) => Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to load site replication repair state: {err}"), - )), - } -} - -async fn save_site_replication_repair_state_to_store(store: Arc, state: &SiteReplicationRepairState) -> S3Result<()> { - let data = serde_json::to_vec(state) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair state failed: {e}")))?; - save_config_no_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH, data) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save repair state failed: {e}"))) -} - -async fn read_site_replication_repair_state() -> S3Result { - let store = - current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - let read_store = store.clone(); - with_config_object_read_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH.to_string(), move || async move { - load_site_replication_repair_state_from_store(read_store).await - }) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock repair state failed: {e}")))? -} - -async fn update_site_replication_repair_state(update: F) -> S3Result -where - T: Send + 'static, - F: FnOnce(&mut SiteReplicationRepairState) -> S3Result + Send + 'static, -{ - let store = - current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - let read_store = store.clone(); - let save_store = store.clone(); - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH.to_string(), move || async move { - let mut state = load_site_replication_repair_state_from_store(read_store).await?; - let result = update(&mut state)?; - save_site_replication_repair_state_to_store(save_store, &state).await?; - Ok(result) - }) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock repair state failed: {e}")))? -} - -/// Test-only seeding of the state object. Every production write goes through -/// [`update_site_replication_state`] — this helper is `cfg(test)` so a new -/// call site cannot reintroduce the pre-P1-15 shape (load through one object -/// lock, save through another, with the mutation in between unprotected). -#[cfg(test)] -async fn save_site_replication_state(state: &SiteReplicationState) -> S3Result<()> { - let Some(store) = current_object_store_handle() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let mut normalized = state.clone(); - normalized.peers = normalize_peer_map_by_identity(normalized.peers); - - let data = serde_json::to_vec(&normalized) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize state failed: {e}")))?; - save_admin_config(store, SITE_REPLICATION_STATE_PATH, data) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save state failed: {e}")))?; - Ok(()) -} - -fn build_site_replication_peer_client(outbound_tls: &GlobalPublishedOutboundTlsState) -> S3Result { - build_site_replication_peer_client_with_resolver(outbound_tls, PeerDnsResolver::new(loopback_replication_targets_allowed())) -} - -fn build_site_replication_peer_client_with_resolver( - outbound_tls: &GlobalPublishedOutboundTlsState, - resolver: PeerDnsResolver, -) -> S3Result { - let mut builder = reqwest::Client::builder() - .no_proxy() - .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT) - .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT) - .pool_idle_timeout(Some(Duration::from_secs(60))) - .redirect(reqwest::redirect::Policy::none()) - .dns_resolver(resolver); - - if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { - let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); - let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to parse published site-replication CA certs: {e}"), - ) - })?; - - for cert_der in certs_der { - let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to load published site-replication CA cert: {e}"), - ) - })?; - builder = builder.add_root_certificate(cert); - } - } - - builder - .build() - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}"))) -} - -fn build_custom_site_replication_peer_client( - outbound_tls: &GlobalPublishedOutboundTlsState, - connection: &PeerConnection, -) -> S3Result { - build_custom_site_replication_peer_client_with_resolver( - outbound_tls, - connection, - PeerDnsResolver::new(loopback_replication_targets_allowed()), - ) -} - -fn build_custom_site_replication_peer_client_with_resolver( - outbound_tls: &GlobalPublishedOutboundTlsState, - connection: &PeerConnection, - resolver: PeerDnsResolver, -) -> S3Result { - let mut builder = reqwest::Client::builder() - .no_proxy() - .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT) - .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT) - .pool_idle_timeout(Some(Duration::from_secs(60))) - .redirect(reqwest::redirect::Policy::none()) - .dns_resolver(resolver) - .danger_accept_invalid_certs(connection.skip_tls_verify); - - if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { - let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); - let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to parse published site-replication CA certs: {e}"), - ) - })?; - for cert_der in certs_der { - let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to load published site-replication CA cert: {e}"), - ) - })?; - builder = builder.add_root_certificate(cert); - } - } - if !connection.ca_cert_pem.is_empty() { - for cert in parse_peer_ca_certificates(&connection.ca_cert_pem)? { - builder = builder.add_root_certificate(cert); - } - } - - builder - .build() - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}"))) -} - -async fn site_replication_peer_client() -> S3Result { - let generation = current_outbound_tls_generation().0; - let cache = SITE_REPLICATION_PEER_CLIENT.lock().await; - if let Some(hit) = site_replication_peer_client_cache_hit(&cache, generation) { - return hit; - } - drop(cache); - - let outbound_tls = current_outbound_tls_state().await; - let built = build_site_replication_peer_client(&outbound_tls); - let cache_entry = match &built { - Ok(client) => SiteReplicationPeerClientCacheEntry::Ready(client.clone()), - Err(err) => SiteReplicationPeerClientCacheEntry::Failed(err.to_string()), - }; - - let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; - if cache.as_ref().is_none_or(|cached| cached.generation <= generation) { - *cache = Some(SiteReplicationPeerClientCache { - generation, - entry: cache_entry, - }); - } - - built -} - -async fn site_replication_client_for(connection: &PeerConnection) -> S3Result { - // Revalidate at the client boundary so callers cannot bypass endpoint/TLS policy. - let connection = PeerConnection::new(connection.endpoint(), connection.skip_tls_verify, &connection.ca_cert_pem)?; - if connection.uses_default_tls() { - return site_replication_peer_client().await; - } - let outbound_tls = current_outbound_tls_state().await; - build_custom_site_replication_peer_client(&outbound_tls, &connection) -} - -fn runtime_peer_connection(peer: &PeerInfo) -> S3Result { - PeerConnection::try_from(peer).map_err(|err| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("invalid persisted site replication peer `{}`: {err}", peer.endpoint), - ) - }) -} - -struct PeerTransport { - connection: PeerConnection, - client: reqwest::Client, -} - -impl PeerTransport { - async fn for_runtime_peer(peer: &PeerInfo) -> S3Result { - let connection = runtime_peer_connection(peer)?; - let client = site_replication_client_for(&connection).await.map_err(|err| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("initialize persisted site replication peer `{}` transport failed: {err}", peer.endpoint), - ) - })?; - Ok(Self { connection, client }) - } -} - -fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_api::runtime::EndpointServerPools>) -> bool { - if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() { - return true; - } - - if let Some(tls_enabled) = endpoints.and_then(|endpoints| { - endpoints - .as_ref() - .iter() - .flat_map(|pool| pool.endpoints.as_ref().iter()) - .find(|endpoint| endpoint.is_local) - .map(|endpoint| endpoint.url.scheme().eq_ignore_ascii_case("https")) - }) { - return tls_enabled; - } - - false -} - -fn runtime_tls_enabled() -> bool { - let endpoints = current_endpoints_handle(); - runtime_tls_enabled_with(endpoints.as_ref()) -} - fn query_pairs(uri: &Uri) -> HashMap { uri.query() .map(|query| { @@ -1625,16 +698,6 @@ fn sr_edit_ilm_expiry_override(uri: &Uri) -> Option { } } -fn hash_client_secret(secret: Option<&str>) -> String { - let Some(secret) = secret.filter(|secret| !secret.is_empty()) else { - return String::new(); - }; - - let mut hasher = Sha256::new(); - hasher.update(secret.as_bytes()); - URL_SAFE_NO_PAD.encode_to_string(hasher.finalize()) -} - fn config_enabled(value: Option) -> bool { matches!(value.as_deref(), Some("on" | "true" | "enabled")) } @@ -1688,96 +751,6 @@ fn load_ldap_idp_settings() -> (LDAPSettings, LDAPConfigSettings) { .unwrap_or_else(|| (LDAPSettings::default(), LDAPConfigSettings::default())) } -fn request_endpoint(uri: &Uri, headers: &HeaderMap) -> String { - let scheme = get_source_scheme(headers) - .and_then(|value| { - value - .split(',') - .next() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_ascii_lowercase) - }) - .or_else(|| uri.scheme_str().map(str::to_ascii_lowercase)) - .unwrap_or_else(|| { - if runtime_tls_enabled() { - "https".to_string() - } else { - "http".to_string() - } - }); - - let host = headers - .get(http::header::HOST) - .and_then(|value| value.to_str().ok()) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .or_else(|| uri.authority().map(|value| value.as_str().to_string())) - .or_else(|| { - current_endpoints_handle().and_then(|endpoints| { - endpoints - .as_ref() - .iter() - .flat_map(|pool| pool.endpoints.as_ref().iter()) - .find(|endpoint| endpoint.is_local) - .map(|endpoint| endpoint.host_port()) - }) - }) - .unwrap_or_else(|| format!("127.0.0.1:{}", current_runtime_port())); - - format!("{scheme}://{host}") -} - -fn runtime_console_port() -> Option { - let console_address = get_config_snapshot() - .map(|snapshot| snapshot.console_address.clone()) - .unwrap_or_else(|| rustfs_utils::get_env_str(ENV_RUSTFS_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ADDRESS)); - - let parse_target = if console_address.starts_with(':') { - format!("127.0.0.1{console_address}") - } else { - console_address - }; - - Url::parse(&format!("http://{parse_target}")) - .ok() - .and_then(|parsed| parsed.port_or_known_default()) -} - -fn site_replication_local_endpoint(uri: &Uri, headers: &HeaderMap) -> String { - let endpoint = request_endpoint(uri, headers); - match Url::parse(&endpoint) { - Ok(mut parsed) => { - if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { - return request_endpoint(&Uri::from_static("/"), &HeaderMap::new()); - } - if parsed.port_or_known_default() == runtime_console_port() && parsed.set_port(Some(current_runtime_port())).is_ok() { - parsed.to_string().trim_end_matches('/').to_string() - } else { - endpoint - } - } - Err(_) => request_endpoint(&Uri::from_static("/"), &HeaderMap::new()), - } -} - -fn current_local_runtime_endpoint() -> String { - site_replication_local_endpoint(&Uri::from_static("/"), &HeaderMap::new()) -} - -fn infer_site_name(endpoint: &str) -> String { - endpoint - .trim_start_matches("http://") - .trim_start_matches("https://") - .split('/') - .next() - .unwrap_or_default() - .split(':') - .next() - .unwrap_or_default() - .to_string() -} - fn qstat(count: i64, bytes: i64) -> QStat { QStat { count: count as f64, @@ -1789,54 +762,10 @@ fn non_negative_u64(value: i64) -> u64 { value.max(0) as u64 } -fn stored_peer_tls_settings(stored_peer: Option<&PeerInfo>) -> (bool, String) { - stored_peer - .map(|peer| (peer.skip_tls_verify, peer.ca_cert_pem.clone())) - .unwrap_or_default() -} - fn current_local_peer(req: &S3Request, state: &SiteReplicationState) -> PeerInfo { local_peer_at_endpoint(site_replication_local_endpoint(&req.uri, &req.headers), state) } -/// The local peer record as the given state describes it. Split out of -/// [`current_local_peer`] so a state transaction can rebuild it against the -/// state it just loaded: the request the endpoint came from cannot cross into -/// the transaction closure, but the endpoint itself can. -fn local_peer_at_endpoint(endpoint: String, state: &SiteReplicationState) -> PeerInfo { - let deployment_id = current_deployment_id().unwrap_or_else(|| deployment_id_for_endpoint(&endpoint)); - let stored_peer = state.peers.get(&deployment_id); - let (skip_tls_verify, ca_cert_pem) = stored_peer_tls_settings(stored_peer); - - PeerInfo { - endpoint: endpoint.clone(), - name: if state.name.is_empty() { - stored_peer - .map(|peer| peer.name.clone()) - .filter(|name| !name.is_empty()) - .unwrap_or_else(|| infer_site_name(&endpoint)) - } else { - state.name.clone() - }, - deployment_id, - sync_state: stored_peer.map(|peer| peer.sync_state.clone()).unwrap_or(SyncStatus::Unknown), - default_bandwidth: stored_peer.map(|peer| peer.default_bandwidth.clone()).unwrap_or_default(), - replicate_ilm_expiry: stored_peer.is_some_and(|peer| peer.replicate_ilm_expiry), - object_naming_mode: stored_peer.map(|peer| peer.object_naming_mode.clone()).unwrap_or_default(), - skip_tls_verify, - ca_cert_pem, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - } -} - -fn current_local_runtime_peer(state: &SiteReplicationState) -> PeerInfo { - local_peer_at_endpoint(current_local_runtime_endpoint(), state) -} - -fn normalize_peer_map_by_identity(peers: BTreeMap) -> BTreeMap { - normalize_peer_map_by_identity_with(peers, normalize_peer_info) -} - fn existing_peer_for_endpoint(state: &SiteReplicationState, endpoint: &str) -> Option { state .peers @@ -1873,19 +802,6 @@ fn peer_deployment_id_for_endpoint(state: &SiteReplicationState, endpoint: &str) .filter(|deployment_id| !deployment_id.is_empty()) } -fn normalize_peer_info(mut peer: PeerInfo) -> PeerInfo { - if peer.deployment_id.is_empty() { - peer.deployment_id = deployment_id_for_endpoint(&peer.endpoint); - } - if peer.name.is_empty() { - peer.name = infer_site_name(&peer.endpoint); - } - if peer.api_version.is_none() { - peer.api_version = Some(SITE_REPL_API_VERSION.to_string()); - } - peer -} - fn normalize_peer_site(site: PeerSite, replicate_ilm_expiry: bool) -> PeerInfo { normalize_peer_info(PeerInfo { endpoint: site.endpoint, @@ -1901,155 +817,6 @@ fn normalize_peer_site(site: PeerSite, replicate_ilm_expiry: bool) -> PeerInfo { }) } -fn loopback_replication_targets_allowed() -> bool { - std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV) - .map(|value| value.eq_ignore_ascii_case("true") || value == "1") - .unwrap_or(false) -} - -fn validate_peer_egress(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> { - match validate_outbound_url(url) { - Ok(()) => Ok(()), - Err(OutboundUrlError::ForbiddenHost { - reason: "private address", - .. - }) => Ok(()), - Err(OutboundUrlError::ForbiddenHost { - reason: "loopback address" | "loopback host", - .. - }) if allow_loopback && peer_url_has_canonical_loopback_host(url) => Ok(()), - Err(err) => Err(err), - } -} - -fn peer_url_has_canonical_loopback_host(url: &Url) -> bool { - match url.host() { - Some(url::Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), - Some(url::Host::Ipv4(ip)) => ip == std::net::Ipv4Addr::LOCALHOST, - Some(url::Host::Ipv6(ip)) => ip == std::net::Ipv6Addr::LOCALHOST, - None => false, - } -} - -fn resolved_peer_ip_allowed(host: &str, ip: IpAddr, allow_loopback: bool) -> bool { - let Ok(ip_url) = (match ip { - IpAddr::V4(ip) => Url::parse(&format!("http://{ip}")), - IpAddr::V6(ip) => Url::parse(&format!("http://[{ip}]")), - }) else { - return false; - }; - match validate_outbound_url(&ip_url) { - Ok(()) => true, - Err(OutboundUrlError::ForbiddenHost { - reason: "private address", - .. - }) => true, - Err(OutboundUrlError::ForbiddenHost { - reason: "loopback address", - .. - }) => { - allow_loopback - && host.eq_ignore_ascii_case("localhost") - && matches!(ip, IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) | IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)) - } - Err(_) => false, - } -} - -fn parse_peer_ca_certificates(ca_cert_pem: &str) -> S3Result> { - if ca_cert_pem.len() > MAX_PEER_CA_CERT_PEM_SIZE { - return Err(s3_error!(InvalidRequest, "site replication CA certificate exceeds 256 KiB")); - } - if ca_cert_pem.contains("PRIVATE KEY-----") { - return Err(s3_error!( - InvalidRequest, - "site replication CA certificate must not contain a private key" - )); - } - - let mut reader = std::io::BufReader::new(ca_cert_pem.as_bytes()); - let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|e| { - S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication CA certificate: {e}")) - })?; - if certs_der.is_empty() { - return Err(s3_error!( - InvalidRequest, - "site replication CA certificate must contain at least one certificate" - )); - } - - let mut root_store = rustls::RootCertStore::empty(); - certs_der - .into_iter() - .map(|cert| { - root_store.add(cert.clone()).map_err(|e| { - S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication CA certificate: {e}")) - })?; - reqwest::Certificate::from_der(cert.as_ref()).map_err(|e| { - S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication CA certificate: {e}")) - }) - }) - .collect() -} - -fn validate_peer_connection_inner( - endpoint: &str, - skip_tls_verify: bool, - ca_cert_pem: &str, - allow_loopback: bool, -) -> S3Result { - let parsed = Url::parse(endpoint) - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site endpoint `{endpoint}`: {e}")))?; - match parsed.scheme() { - "http" | "https" => {} - scheme => { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("invalid site endpoint `{endpoint}`: unsupported scheme `{scheme}`"), - )); - } - } - if parsed.host_str().is_none() { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("invalid site endpoint `{endpoint}`: missing host"), - )); - } - if !parsed.username().is_empty() || parsed.password().is_some() { - return Err(s3_error!(InvalidRequest, "invalid site endpoint `{endpoint}`: userinfo is not allowed")); - } - if parsed.path() != "/" || parsed.query().is_some() || parsed.fragment().is_some() { - return Err(s3_error!( - InvalidRequest, - "invalid site endpoint `{endpoint}`: endpoint must be an origin" - )); - } - validate_peer_egress(&parsed, allow_loopback) - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site endpoint `{endpoint}`: {e}")))?; - - if ca_cert_pem.len() > MAX_PEER_CA_CERT_PEM_SIZE { - return Err(s3_error!(InvalidRequest, "site replication CA certificate exceeds 256 KiB")); - } - let ca_cert_pem = ca_cert_pem.trim(); - if parsed.scheme() != "https" && (skip_tls_verify || !ca_cert_pem.is_empty()) { - return Err(s3_error!(InvalidRequest, "site replication TLS settings require an HTTPS endpoint")); - } - if skip_tls_verify && !ca_cert_pem.is_empty() { - return Err(s3_error!(InvalidRequest, "skipTLSVerify and caCertPem are mutually exclusive")); - } - if !ca_cert_pem.is_empty() { - parse_peer_ca_certificates(ca_cert_pem)?; - } - - Ok(PeerConnection { - endpoint: parsed, - skip_tls_verify, - ca_cert_pem: ca_cert_pem.to_string(), - }) -} - fn validate_proposed_peer(peer: &PeerInfo) -> S3Result<()> { PeerConnection::try_from(peer).map(|_| ()) } @@ -2332,305 +1099,10 @@ fn validate_add_preflight_topology(infos: &[SiteReplicationAddPreflightInfo], lo Ok(()) } -fn bootstrap_bucket_op_path(bucket: &str, operation: &str) -> String { - format!( - "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", - form_urlencoded::Serializer::new(String::new()) - .append_pair("bucket", bucket) - .append_pair("operation", operation) - .finish() - ) -} - -fn with_site_replication_bootstrap_token(path: &str, token: &str) -> String { - let separator = if path.contains('?') { '&' } else { '?' }; - let query = form_urlencoded::Serializer::new(String::new()) - .append_pair("bootstrapToken", token) - .finish(); - format!("{path}{separator}{query}") -} - fn site_replication_bootstrap_token(uri: &Uri) -> Option { query_pairs(uri).get("bootstrapToken").cloned() } -/// Query for a peer `make-with-versioning` bucket op. `versioningEnabled` -/// always travels so the outbound query matches MinIO's site-replication -/// make-bucket wire contract: MinIO's own create-bucket hook sends -/// `versioningEnabled=true` on this op. RustFS's inbound handler -/// force-enables versioning either way. -fn make_with_versioning_bucket_op_path(bucket: &str, created_at: Option<&str>, lock_enabled: bool) -> String { - let mut query = form_urlencoded::Serializer::new(String::new()); - query.append_pair("bucket", bucket); - query.append_pair("operation", SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING); - query.append_pair("versioningEnabled", "true"); - if let Some(created_at) = created_at { - query.append_pair("createdAt", created_at); - } - if lock_enabled { - query.append_pair("lockEnabled", "true"); - } - format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?{}", query.finish()) -} - -fn bootstrap_bucket_make_op_path(bucket: &SRBucketInfo) -> String { - let created_at = bucket - .created_at - .and_then(|value| value.format(&time::format_description::well_known::Rfc3339).ok()); - make_with_versioning_bucket_op_path(&bucket.bucket, created_at.as_deref(), bucket.object_lock_config.is_some()) -} - -fn bootstrap_bucket_meta_item(bucket: &SRBucketInfo, item_type: &str, updated_at: Option) -> SRBucketMeta { - SRBucketMeta { - bucket: bucket.bucket.clone(), - r#type: item_type.to_string(), - updated_at, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - derived_rule_contract: true, - ..Default::default() - } -} - -fn bootstrap_bucket_quota_value(bucket: &str, raw: &str) -> S3Result { - serde_json::from_slice(&decode_bucket_meta_wire_value(raw)) - .map_err(|e| s3_error!(InvalidRequest, "invalid quota metadata for bootstrap bucket `{bucket}`: {e}")) -} - -fn append_bootstrap_bucket_item( - items: &mut Vec, - bucket: &SRBucketInfo, - item_type: &str, - value: Option, - updated_at: Option, - apply: impl FnOnce(&mut SRBucketMeta, String) -> S3Result<()>, -) -> S3Result<()> { - if let Some(value) = value { - let mut item = bootstrap_bucket_meta_item(bucket, item_type, updated_at); - apply(&mut item, value)?; - items.push(item); - } - Ok(()) -} - -fn append_bootstrap_bucket_items( - plan: &mut SiteReplicationBootstrapPlan, - bucket: &SRBucketInfo, - replicate_ilm_expiry: bool, -) -> S3Result<()> { - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "policy", - bucket.policy.clone().map(|value| value.to_string()), - bucket.policy_updated_at, - |item, value| { - item.policy = - Some(serde_json::from_str(&value).map_err(|e| { - s3_error!(InvalidRequest, "invalid bucket policy for bootstrap bucket `{}`: {e}", item.bucket) - })?); - Ok(()) - }, - )?; - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "version-config", - bucket.versioning.clone(), - bucket.versioning_config_updated_at, - |item, value| { - item.versioning = Some(value); - Ok(()) - }, - )?; - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "tags", - bucket.tags.clone(), - bucket.tag_config_updated_at, - |item, value| { - item.tags = Some(value); - Ok(()) - }, - )?; - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "object-lock-config", - bucket.object_lock_config.clone(), - bucket.object_lock_config_updated_at, - |item, value| { - item.object_lock_config = Some(value); - Ok(()) - }, - )?; - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "sse-config", - bucket.sse_config.clone(), - bucket.sse_config_updated_at, - |item, value| { - item.sse_config = Some(value); - Ok(()) - }, - )?; - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "replication-config", - bucket.replication_config.clone(), - bucket.replication_config_updated_at, - |item, value| { - item.replication_config = Some(value); - Ok(()) - }, - )?; - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "quota-config", - bucket.quota_config.clone(), - bucket.quota_config_updated_at, - |item, value| { - item.quota = Some(bootstrap_bucket_quota_value(&item.bucket, &value)?); - Ok(()) - }, - )?; - if replicate_ilm_expiry { - if bucket.expiry_lc_config.is_some() { - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "lc-config", - bucket.expiry_lc_config.clone(), - bucket.expiry_lc_config_updated_at, - |item, value| { - item.expiry_lc_config = Some(value); - // `updated_at` here is the entry's expiry axis (see the - // SRBucketInfo construction), not the wall clock. - item.expiry_updated_at = item.updated_at; - Ok(()) - }, - )?; - } else if bucket.expiry_lc_config_updated_at.is_some() { - // Expiry rules were removed at this axis (lifecycle_expiry_statement): - // an explicit timestamped delete item, so a peer that missed the - // live delete converges on bootstrap/repair instead of keeping - // stale expiry rules. The receiver's staleness guard protects a - // peer whose expiry state is newer. - let mut item = bootstrap_bucket_meta_item(bucket, "lc-config", bucket.expiry_lc_config_updated_at); - item.expiry_updated_at = item.updated_at; - plan.bucket_items.push(item); - } - } - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "cors-config", - bucket.cors_config.clone(), - bucket.cors_config_updated_at, - |item, value| { - item.cors = Some(value); - Ok(()) - }, - ) -} - -fn group_status_from_desc(status: &str) -> GroupStatus { - if status.eq_ignore_ascii_case("disabled") { - GroupStatus::Disabled - } else { - GroupStatus::Enabled - } -} - -fn site_replication_info_replicates_ilm_expiry(info: &SRInfo) -> bool { - info.state.peers.values().any(|peer| peer.replicate_ilm_expiry) -} - -fn site_replication_state_replicates_ilm_expiry(state: &SiteReplicationState) -> bool { - state.peers.values().any(|peer| peer.replicate_ilm_expiry) -} - -fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result { - let mut plan = SiteReplicationBootstrapPlan::default(); - let replicate_ilm_expiry = site_replication_info_replicates_ilm_expiry(info); - - for (name, policy) in &info.policies { - plan.iam_items.push(SRIAMItem { - r#type: "policy".to_string(), - name: name.clone(), - policy: policy.policy.clone(), - updated_at: policy.updated_at, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }); - } - - for (access_key, user) in &info.user_info_map { - if let Some(secret_key) = &user.secret_key { - plan.iam_items.push(SRIAMItem { - r#type: "iam-user".to_string(), - iam_user: Some(rustfs_madmin::SRIAMUser { - access_key: access_key.clone(), - is_delete_req: false, - user_req: Some(AddOrUpdateUserReq { - secret_key: secret_key.clone(), - policy: user.policy_name.clone(), - status: user.status.clone(), - }), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }), - updated_at: user.updated_at, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }); - } - } - - for (name, desc) in &info.group_desc_map { - plan.iam_items.push(SRIAMItem { - r#type: "group-info".to_string(), - group_info: Some(SRGroupInfo { - update_req: GroupAddRemove { - group: if desc.name.is_empty() { - name.clone() - } else { - desc.name.clone() - }, - members: desc.members.clone(), - status: group_status_from_desc(&desc.status), - is_remove: false, - }, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }), - updated_at: desc.updated_at, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }); - } - - for mapping in info.user_policies.values().chain(info.group_policies.values()) { - plan.iam_items.push(SRIAMItem { - r#type: "policy-mapping".to_string(), - policy_mapping: Some(mapping.clone()), - updated_at: mapping.updated_at, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }); - } - - for bucket in info.buckets.values() { - plan.bucket_make_ops.push(bootstrap_bucket_make_op_path(bucket)); - append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?; - plan.bucket_configure_ops - .push(bootstrap_bucket_op_path(&bucket.bucket, "configure-replication")); - } - - Ok(plan) -} - fn build_join_peers( state: &SiteReplicationState, local_peer: &PeerInfo, @@ -2782,23 +1254,6 @@ fn reconcile_peer_with_actual_identity(mut state: SiteReplicationState, actual_p state } -async fn site_replicator_service_account_secret(access_key: &str) -> S3Result { - let Some(iam_sys) = current_iam_handle() else { - return Err(s3_error!(InvalidRequest, "iam not init")); - }; - - iam_sys - .get_site_replicator_service_account_secret(access_key) - .await - .map_err(ApiError::from) - .map_err(Into::into) -} - -fn legacy_site_replicator_state_secret(state: &SiteReplicationState) -> Option { - (state.service_account_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT && !state.service_account_secret_key.is_empty()) - .then(|| state.service_account_secret_key.clone()) -} - async fn set_site_replicator_service_account_secret(parent_user: &str, secret_key: String) -> S3Result { let Some(iam_sys) = current_iam_handle() else { return Err(s3_error!(InvalidRequest, "iam not init")); @@ -3234,390 +1689,6 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin String { - let (path_only, query) = path - .split_once('?') - .map(|(path, query)| (path, Some(query))) - .unwrap_or((path, None)); - let wire_path = if let Some(suffix) = path_only.strip_prefix(RUSTFS_ADMIN_V3_PREFIX) { - format!("{MINIO_ADMIN_V3_PREFIX}{suffix}") - } else { - path_only.to_string() - }; - - match query { - Some(query) => format!("{wire_path}?{query}"), - None => wire_path, - } -} - -fn site_replication_peer_payload_encrypted(wire_path: &str) -> bool { - // MinIO's SRPeerJoin handler force-decrypts the request body, so the - // peer/join payload must always travel encrypted. - wire_path.split_once('?').map(|(path, _)| path).unwrap_or(wire_path) == MINIO_SITE_REPLICATION_PEER_JOIN_PATH -} - -fn site_replication_peer_payload(path: &str, secret_key: &str, payload: Vec) -> S3Result<(Vec, &'static str)> { - if site_replication_peer_payload_encrypted(path) { - encode_compatible_admin_payload(path, secret_key, payload) - } else { - Ok((payload, "application/json")) - } -} - -fn site_replication_peer_url(connection: &PeerConnection, wire_path: &str) -> S3Result { - let path = wire_path.split_once('?').map_or(wire_path, |(path, _)| path); - if !path.starts_with('/') || path.starts_with("//") { - return Err(s3_error!(InvalidRequest, "invalid site replication peer path")); - } - connection - .endpoint - .join(wire_path) - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication peer path: {e}"))) -} - -#[cfg(test)] -async fn send_peer_admin_request_raw( - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_key: &str, - body: &T, -) -> S3Result<(StatusCode, Vec)> { - let client = site_replication_client_for(connection).await?; - send_peer_admin_request_raw_with_client(&client, connection, path, access_key, secret_key, body).await -} - -async fn send_peer_admin_request_raw_with_client( - client: &reqwest::Client, - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_key: &str, - body: &T, -) -> S3Result<(StatusCode, Vec)> { - let path = site_replication_peer_wire_path(path); - let url = site_replication_peer_url(connection, &path)?; - let uri = url - .as_str() - .parse::() - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {e}")))?; - let authority = uri - .authority() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "peer endpoint missing authority".to_string()))? - .to_string(); - let payload = serde_json::to_vec(body) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize peer request failed: {e}")))?; - let (payload, content_type) = site_replication_peer_payload(&path, secret_key, payload)?; - - let signed = sign_v4( - http::Request::builder() - .method(Method::PUT) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) - .header(CONTENT_TYPE, content_type) - .body(Body::empty()) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build peer request failed: {e}")))?, - payload.len() as i64, - access_key, - secret_key, - "", - current_region() - .map(|region| region.to_string()) - .as_deref() - .unwrap_or("us-east-1"), - ); - - let mut req = client.request(reqwest::Method::PUT, url.clone()); - for (name, value) in signed.headers() { - req = req.header(name, value); - } - - let response = req.body(payload).send().await.map_err(|e| { - let classify = if e.is_timeout() { - "timeout" - } else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") { - "dns resolution" - } else if e.to_string().to_ascii_lowercase().contains("certificate") || e.to_string().to_ascii_lowercase().contains("tls") - { - "tls handshake" - } else if e.is_connect() { - "connect" - } else { - "request" - }; - S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) - })?; - - let status = response.status(); - let body = response - .bytes() - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("read peer response failed: {e}")))?; - - Ok((status, body.to_vec())) -} - -async fn send_peer_admin_request( - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_key: &str, - body: &T, -) -> S3Result> { - let client = site_replication_client_for(connection).await?; - send_peer_admin_request_with_client(&client, connection, path, access_key, secret_key, body).await -} - -async fn send_peer_admin_request_with_client( - client: &reqwest::Client, - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_key: &str, - body: &T, -) -> S3Result> { - let (status, body) = send_peer_admin_request_raw_with_client(client, connection, path, access_key, secret_key, body).await?; - if status.is_success() { - return Ok(body); - } - - let detail = String::from_utf8_lossy(&body).into_owned(); - Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("peer request to {}{path} failed with {status}: {detail}", connection.endpoint()), - )) -} - -async fn send_peer_admin_request_with_secret_candidates( - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_candidates: &[String], - body: &T, -) -> S3Result> { - let client = site_replication_client_for(connection).await?; - let mut tried = HashSet::new(); - let mut errors = Vec::new(); - - for secret_key in secret_candidates.iter().filter(|secret_key| !secret_key.is_empty()) { - if !tried.insert(secret_key.as_str()) { - continue; - } - - match send_peer_admin_request_with_client(&client, connection, path, access_key, secret_key, body).await { - Ok(body) => return Ok(body), - Err(err) => { - let detail = format!("{err}"); - let may_retry_with_next_secret = peer_error_may_be_secret_mismatch(&detail); - errors.push(summarize_peer_error_detail(&detail)); - if !may_retry_with_next_secret { - break; - } - } - } - } - - Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!( - "peer request to {}{path} failed with all service-account secrets: {}", - connection.endpoint(), - errors.join("; ") - ), - )) -} - -fn peer_error_may_be_secret_mismatch(detail: &str) -> bool { - let detail = detail.to_ascii_lowercase(); - detail.contains("signaturedoesnotmatch") - || detail.contains("accessdenied") - || detail.contains("forbidden") - || detail.contains("401") - || detail.contains("403") -} - -async fn send_peer_admin_get_request( - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_key: &str, -) -> S3Result> { - let client = site_replication_client_for(connection).await?; - send_peer_admin_get_request_with_client(&client, connection, path, access_key, secret_key).await -} - -async fn send_peer_admin_get_request_with_client( - client: &reqwest::Client, - connection: &PeerConnection, - path: &str, - access_key: &str, - secret_key: &str, -) -> S3Result> { - let path = site_replication_peer_wire_path(path); - let url = site_replication_peer_url(connection, &path)?; - let uri = url - .as_str() - .parse::() - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {e}")))?; - let authority = uri - .authority() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "peer endpoint missing authority".to_string()))? - .to_string(); - - let signed = sign_v4( - http::Request::builder() - .method(Method::GET) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) - .body(Body::empty()) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build peer request failed: {e}")))?, - 0, - access_key, - secret_key, - "", - current_region() - .map(|region| region.to_string()) - .as_deref() - .unwrap_or("us-east-1"), - ); - - let mut req = client.request(reqwest::Method::GET, url.clone()); - for (name, value) in signed.headers() { - req = req.header(name, value); - } - - let response = req.send().await.map_err(|e| { - let classify = if e.is_timeout() { - "timeout" - } else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") { - "dns resolution" - } else if e.to_string().to_ascii_lowercase().contains("certificate") || e.to_string().to_ascii_lowercase().contains("tls") - { - "tls handshake" - } else if e.is_connect() { - "connect" - } else { - "request" - }; - S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) - })?; - - let status = response.status(); - let body = response - .bytes() - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("read peer response failed: {e}")))?; - - if !status.is_success() { - let detail = String::from_utf8_lossy(&body).into_owned(); - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("peer request to {url} failed with {status}: {detail}"), - )); - } - - Ok(body.to_vec()) -} - -async fn runtime_site_replication_targets() -> S3Result> { - let state = load_site_replication_state().await?; - if !state.enabled() || state.service_account_access_key.is_empty() { - return Ok(None); - } - - let service_account_secret_key = match site_replicator_service_account_secret(&state.service_account_access_key).await { - Ok(secret) => secret, - Err(err) => { - let Some(secret) = legacy_site_replicator_state_secret(&state) else { - return Err(err); - }; - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - result = "legacy_state_service_account_secret_fallback", - error = ?err, - "admin site replication state" - ); - secret - } - }; - let local_peer = current_local_runtime_peer(&state); - Ok(Some(SiteReplicationRuntime { - state, - local_peer, - service_account_secret_key, - })) -} - -async fn broadcast_site_replication_json(path: &str, body: &T) -> S3Result<()> { - let Some(runtime) = runtime_site_replication_targets().await? else { - return Ok(()); - }; - broadcast_site_replication_json_with_runtime(&runtime, path, body).await -} - -async fn broadcast_site_replication_json_with_runtime( - runtime: &SiteReplicationRuntime, - path: &str, - body: &T, -) -> S3Result<()> { - let state = &runtime.state; - let local_peer = &runtime.local_peer; - - for peer in state.peers.values() { - if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { - continue; - } - - send_peer_admin_request_with_retry_event( - peer, - path, - &state.service_account_access_key, - &runtime.service_account_secret_key, - body, - ) - .await?; - } - - Ok(()) -} - -async fn send_peer_admin_request_with_retry_event( - peer: &PeerInfo, - path: &str, - access_key: &str, - secret_key: &str, - body: &T, -) -> S3Result> { - let transport = PeerTransport::for_runtime_peer(peer).await?; - send_peer_admin_request_with_retry_event_transport(peer, &transport, path, access_key, secret_key, body).await -} - -async fn send_peer_admin_request_with_retry_event_transport( - peer: &PeerInfo, - transport: &PeerTransport, - path: &str, - access_key: &str, - secret_key: &str, - body: &T, -) -> S3Result> { - match send_peer_admin_request_with_client(&transport.client, &transport.connection, path, access_key, secret_key, body).await - { - Ok(body) => { - dequeue_site_replication_retry_event(peer, path).await; - Ok(body) - } - Err(err) => { - enqueue_site_replication_retry_event(peer, path, &err).await; - Err(err) - } - } -} - async fn send_site_replication_bootstrap_plan( peer: &PeerInfo, service_account_access_key: &str, @@ -3724,915 +1795,6 @@ async fn bootstrap_existing_metadata_after_add( errors } -enum SiteReplicationRepairTask<'a> { - Iam(&'a SRIAMItem), - BucketMake(&'a str), - BucketMetadata(&'a SRBucketMeta), - Replication(&'a str), -} - -impl SiteReplicationRepairTask<'_> { - fn family(&self) -> &'static str { - match self { - Self::Iam(_) => SITE_REPLICATION_REPAIR_IAM_FAMILY, - Self::BucketMake(_) => SITE_REPLICATION_REPAIR_BUCKET_FAMILY, - Self::BucketMetadata(_) => SITE_REPLICATION_REPAIR_BUCKET_METADATA_FAMILY, - Self::Replication(_) => SITE_REPLICATION_REPAIR_REPLICATION_FAMILY, - } - } - - fn path(&self) -> &str { - match self { - Self::Iam(_) => "/rustfs/admin/v3/site-replication/peer/iam-item", - Self::BucketMake(path) | Self::Replication(path) => path, - Self::BucketMetadata(_) => "/rustfs/admin/v3/site-replication/peer/bucket-meta", - } - } - - fn id(&self) -> S3Result { - let payload = match self { - Self::Iam(item) => serde_json::to_vec(item), - Self::BucketMake(_) | Self::Replication(_) => serde_json::to_vec(&serde_json::json!({})), - Self::BucketMetadata(item) => serde_json::to_vec(item), - } - .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair task failed: {err}")))?; - let mut digest = Sha256::new(); - digest.update(self.family().as_bytes()); - digest.update([0]); - digest.update(self.path().as_bytes()); - digest.update([0]); - digest.update(payload); - Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize())) - } - - async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result> { - match self { - Self::Iam(item) => { - send_peer_admin_request_with_client( - &transport.client, - &transport.connection, - self.path(), - access_key, - secret_key, - item, - ) - .await - } - Self::BucketMetadata(item) => { - send_peer_admin_request_with_client( - &transport.client, - &transport.connection, - self.path(), - access_key, - secret_key, - item, - ) - .await - } - Self::BucketMake(_) | Self::Replication(_) => { - send_peer_admin_request_with_client( - &transport.client, - &transport.connection, - self.path(), - access_key, - secret_key, - &serde_json::json!({}), - ) - .await - } - } - } -} - -fn site_replication_repair_tasks(plan: &SiteReplicationBootstrapPlan) -> Vec<(usize, SiteReplicationRepairTask<'_>)> { - let mut tasks = Vec::with_capacity( - plan.iam_items.len() + plan.bucket_make_ops.len() + plan.bucket_items.len() + plan.bucket_configure_ops.len(), - ); - tasks.extend( - plan.iam_items - .iter() - .enumerate() - .map(|(index, item)| (index, SiteReplicationRepairTask::Iam(item))), - ); - tasks.extend( - plan.bucket_make_ops - .iter() - .enumerate() - .map(|(index, path)| (index, SiteReplicationRepairTask::BucketMake(path))), - ); - tasks.extend( - plan.bucket_items - .iter() - .enumerate() - .map(|(index, item)| (index, SiteReplicationRepairTask::BucketMetadata(item))), - ); - tasks.extend( - plan.bucket_configure_ops - .iter() - .enumerate() - .map(|(index, path)| (index, SiteReplicationRepairTask::Replication(path))), - ); - tasks -} - -fn site_replication_repair_plan_token(state: &SiteReplicationState, plan: &SiteReplicationBootstrapPlan) -> S3Result { - let mut digest = Sha256::new(); - let snapshot = serde_json::to_vec(&( - &state.name, - &state.service_account_access_key, - &state.peers, - state.updated_at, - state.sync_state_initialized, - )) - .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair snapshot failed: {err}")))?; - digest.update(snapshot); - for (_, task) in site_replication_repair_tasks(plan) { - digest.update(task.id()?.as_bytes()); - } - Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize())) -} - -fn site_replication_repair_preflight_token( - state: &SiteReplicationState, - plan: &SiteReplicationBootstrapPlan, - signing_key: &[u8], -) -> S3Result { - if signing_key.is_empty() { - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - "repair signing key is empty".to_string(), - )); - } - let mut digest = as hmac::digest::KeyInit>::new_from_slice(signing_key) - .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid repair signing key".to_string()))?; - digest.update(b"rustfs:site-replication:repair-preflight:v1\0"); - digest.update(site_replication_repair_plan_token(state, plan)?.as_bytes()); - for event in state - .retry_queue - .iter() - .filter(|event| retry_event_replayed_by_bootstrap(event)) - { - digest.update(event.id.as_bytes()); - digest.update(&[0]); - digest.update(event.peer_deployment_id.as_bytes()); - digest.update(&[0]); - digest.update(event.path.as_bytes()); - digest.update(&[0]); - } - Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes())) -} - -fn site_replication_repair_task_checkpoint_id( - signing_key: &[u8], - peer_deployment_id: &str, - task: &SiteReplicationRepairTask<'_>, -) -> S3Result { - let mut digest = as hmac::digest::KeyInit>::new_from_slice(signing_key) - .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid repair signing key".to_string()))?; - digest.update(b"rustfs:site-replication:repair-task:v1\0"); - digest.update(peer_deployment_id.as_bytes()); - digest.update(&[0]); - digest.update(task.id()?.as_bytes()); - Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes())) -} - -fn site_replication_repair_sites( - state: &SiteReplicationState, - local_peer: &PeerInfo, - plan: &SiteReplicationBootstrapPlan, - signing_key: &[u8], -) -> S3Result> { - let mut planned = BTreeMap::new(); - let mut family_paths = BTreeMap::>::new(); - for (_, task) in site_replication_repair_tasks(plan) { - let family = task.family().to_string(); - let family_status = planned - .entry(task.family().to_string()) - .or_insert_with(SiteReplicationRepairFamilyStatus::default); - family_status.planned += 1; - family_paths.entry(family).or_default().insert(task.path().to_string()); - } - - let mut sites = BTreeMap::new(); - for peer in state.peers.values().filter(|peer| { - peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) - }) { - let mut families = planned.clone(); - for (_, task) in site_replication_repair_tasks(plan) { - let family = families - .get_mut(task.family()) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair task family is missing".to_string()))?; - family.tasks.push(SiteReplicationRepairTaskStatus { - task_id: site_replication_repair_task_checkpoint_id(signing_key, &peer.deployment_id, &task)?, - status: "planned".to_string(), - error: None, - }); - } - for (family, status) in &mut families { - status.retry_events = state - .retry_queue - .iter() - .filter(|event| { - event.peer_deployment_id == peer.deployment_id - && retry_event_replayed_by_bootstrap(event) - && family_paths.get(family).is_some_and(|paths| paths.contains(&event.path)) - }) - .count(); - } - sites.insert( - peer.deployment_id.clone(), - SiteReplicationRepairSiteStatus { - deployment_id: peer.deployment_id.clone(), - name: peer.name.clone(), - families, - }, - ); - } - Ok(sites) -} - -fn update_site_replication_repair_task( - operation: &mut SiteReplicationRepairOperation, - deployment_id: &str, - family: &str, - family_index: usize, - result: Result<(), &str>, -) -> S3Result<()> { - let site = operation - .sites - .get_mut(deployment_id) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation site is missing".to_string()))?; - let family_status = site - .families - .get_mut(family) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation family is missing".to_string()))?; - if family_status.succeeded != family_index { - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - "repair operation task checkpoint is invalid".to_string(), - )); - } - let task_status = family_status.tasks.get_mut(family_index).ok_or_else(|| { - S3Error::with_message(S3ErrorCode::InternalError, "repair operation task checkpoint is missing".to_string()) - })?; - family_status.failed = 0; - family_status.errors.clear(); - match result { - Ok(()) => { - family_status.succeeded = family_status.succeeded.saturating_add(1); - task_status.status = "succeeded".to_string(); - task_status.error = None; - } - Err(error) => { - let error = classify_site_replication_repair_error(error).to_string(); - family_status.failed = 1; - family_status.errors.push(error.clone()); - task_status.status = "failed".to_string(); - task_status.error = Some(error); - } - } - Ok(()) -} - -fn site_replication_repair_task_pending( - operation: &SiteReplicationRepairOperation, - deployment_id: &str, - family: &str, - family_index: usize, -) -> S3Result { - let site = operation - .sites - .get(deployment_id) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation site is missing".to_string()))?; - let family = site - .families - .get(family) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation family is missing".to_string()))?; - if family.succeeded > family_index { - return Ok(false); - } - if family.succeeded < family_index { - return Ok(false); - } - Ok(family.failed == 0) -} - -fn prepare_site_replication_repair_retry(operation: &mut SiteReplicationRepairOperation) { - for family in operation.sites.values_mut().flat_map(|site| site.families.values_mut()) { - family.failed = 0; - family.errors.clear(); - for task in &mut family.tasks { - match task.status.as_str() { - "succeeded" => task.status = "skipped".to_string(), - "failed" => { - task.status = "planned".to_string(); - task.error = None; - } - _ => {} - } - } - } -} - -fn classify_site_replication_repair_error(error: &str) -> &'static str { - let error = error.to_ascii_lowercase(); - if error.contains("accessdenied") - || error.contains("signaturedoesnotmatch") - || error.contains("unauthorized") - || error.contains("forbidden") - || error.contains("401") - || error.contains("403") - { - "authorization-failed" - } else if error.contains("timeout") { - "remote-timeout" - } else if error.contains("dns") { - "remote-dns-failed" - } else if error.contains("tls") || error.contains("certificate") { - "remote-tls-failed" - } else if error.contains("connect") { - "remote-connect-failed" - } else { - "remote-operation-failed" - } -} - -fn summarize_site_replication_repair_operation(operation: &mut SiteReplicationRepairOperation) { - let failed = operation - .sites - .values() - .flat_map(|site| site.families.values()) - .any(|family| family.failed > 0); - let complete = operation - .sites - .values() - .all(|site| site.families.values().all(|family| family.succeeded == family.planned)); - operation.status = if complete { - "success" - } else if failed { - "partial" - } else { - "running" - } - .to_string(); - operation.updated_at = Some(OffsetDateTime::now_utc()); - operation.completed_at = complete.then_some(OffsetDateTime::now_utc()); -} - -fn site_replication_repair_operation_response( - operation: &SiteReplicationRepairOperation, -) -> SiteReplicationRepairOperationResponse { - SiteReplicationRepairOperationResponse { - mode: "execute", - operation_id: operation.operation_id.clone(), - status: operation.status.clone(), - sites: operation - .sites - .iter() - .map(|(deployment_id, site)| { - ( - deployment_id.clone(), - SiteReplicationRepairSiteResponse { - deployment_id: site.deployment_id.clone(), - name: site.name.clone(), - families: site - .families - .iter() - .map(|(family, status)| { - ( - family.clone(), - SiteReplicationRepairFamilyResponse { - planned: status.planned, - succeeded: status.succeeded, - failed: status.failed, - retry_events: status.retry_events, - tasks: status.tasks.clone(), - errors: status.errors.clone(), - }, - ) - }) - .collect(), - }, - ) - }) - .collect(), - created_at: operation.created_at, - updated_at: operation.updated_at, - completed_at: operation.completed_at, - } -} - -fn prune_site_replication_repair_operations(operations: &mut BTreeMap) { - while operations.len() > SITE_REPLICATION_REPAIR_OPERATION_LIMIT { - let Some(oldest) = operations - .iter() - .filter(|(_, operation)| operation.status == "success") - .min_by_key(|(_, operation)| operation.created_at) - .map(|(id, _)| id.clone()) - else { - break; - }; - operations.remove(&oldest); - } -} - -async fn persist_site_replication_repair_operation(operation: &SiteReplicationRepairOperation) -> S3Result<()> { - let operation = operation.clone(); - update_site_replication_repair_state(move |state| { - if let Some(existing) = state.operations.get(&operation.operation_id) - && !constant_time_eq(&existing.preflight_token, &operation.preflight_token) - { - return Err(S3Error::with_message( - S3ErrorCode::ClientTokenConflict, - "repair operation ID is already bound to a different preflight".to_string(), - )); - } - state.operations.insert(operation.operation_id.clone(), operation); - prune_site_replication_repair_operations(&mut state.operations); - Ok(()) - }) - .await -} - -async fn persist_site_replication_repair_task( - operation: &SiteReplicationRepairOperation, - peer: &PeerInfo, - family: &str, - path: &str, -) -> S3Result<()> { - persist_site_replication_repair_operation(operation).await?; - - let family_status = operation - .sites - .get(&peer.deployment_id) - .and_then(|site| site.families.get(family)) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair task status is missing".to_string()))?; - let failure = (family_status.failed > 0).then(|| { - family_status - .errors - .first() - .cloned() - .unwrap_or_else(|| "remote-operation-failed".to_string()) - }); - let peer = peer.clone(); - let path = path.to_string(); - update_site_replication_state(move |state| { - match failure.as_deref() { - Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), - None => { - dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); - } - } - Ok(()) - }) - .await -} - -fn admit_site_replication_repair_operation( - repair_state: &mut SiteReplicationRepairState, - operation_id: String, - supplied_token: &str, - candidate: SiteReplicationRepairOperation, -) -> S3Result { - if let Some(existing) = repair_state.operations.get(&operation_id) { - if !constant_time_eq(&existing.preflight_token, supplied_token) { - return Err(S3Error::with_message( - S3ErrorCode::ClientTokenConflict, - "repair operation ID is already bound to a different preflight".to_string(), - )); - } - if !constant_time_eq(&existing.plan_token, &candidate.plan_token) { - return Err(S3Error::with_message( - S3ErrorCode::PreconditionFailed, - "site replication repair plan changed after partial execution".to_string(), - )); - } - return Ok(existing.clone()); - } - if repair_state - .operations - .values() - .any(|operation| operation.status == "running") - { - return Err(S3Error::with_message( - S3ErrorCode::ClientTokenConflict, - "another site replication repair is active".to_string(), - )); - } - repair_state.operations.insert(operation_id, candidate.clone()); - prune_site_replication_repair_operations(&mut repair_state.operations); - Ok(candidate) -} - -async fn execute_site_replication_repair( - request: SiteReplicationRepairExecutionRequest, -) -> S3Result> { - let store = - current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { - execute_site_replication_repair_locked(request).await - }) - .await - .map_err(|_| { - S3Error::with_message(S3ErrorCode::ClientTokenConflict, "another site replication repair is active".to_string()) - })? -} - -async fn execute_site_replication_repair_locked( - request: SiteReplicationRepairExecutionRequest, -) -> S3Result> { - let state = load_site_replication_state().await?; - if !state.enabled() || state.service_account_access_key.is_empty() { - return Err(s3_error!(InvalidRequest, "site replication is not configured")); - } - let info = build_sr_info(&state, &request.local_peer).await?; - let plan = site_replication_bootstrap_plan(&info)?; - let plan_token = site_replication_repair_plan_token(&state, &plan)?; - let preflight_token = site_replication_repair_preflight_token(&state, &plan, request.signing_key.as_bytes())?; - let sites = site_replication_repair_sites(&state, &request.local_peer, &plan, request.signing_key.as_bytes())?; - - let repair_state = read_site_replication_repair_state().await?; - if let Some(existing) = repair_state.operations.get(&request.operation_id) { - if !constant_time_eq(&existing.preflight_token, &request.preflight_token) { - return Err(S3Error::with_message( - S3ErrorCode::ClientTokenConflict, - "repair operation ID is already bound to a different preflight".to_string(), - )); - } - if existing.status == "success" { - return json_response(StatusCode::OK, &site_replication_repair_operation_response(existing)); - } - if !constant_time_eq(&existing.plan_token, &plan_token) { - return Err(S3Error::with_message( - S3ErrorCode::PreconditionFailed, - "site replication repair plan changed after partial execution".to_string(), - )); - } - } else if !constant_time_eq(&request.preflight_token, &preflight_token) { - return Err(S3Error::with_message( - S3ErrorCode::PreconditionFailed, - "site replication repair preflight is stale".to_string(), - )); - } - - let now = OffsetDateTime::now_utc(); - let candidate = SiteReplicationRepairOperation { - operation_id: request.operation_id.clone(), - preflight_token, - plan_token, - status: "running".to_string(), - sites, - created_at: Some(now), - updated_at: Some(now), - completed_at: None, - }; - let supplied_token = request.preflight_token; - let operation_id = request.operation_id; - let mut operation = update_site_replication_repair_state(move |repair_state| { - admit_site_replication_repair_operation(repair_state, operation_id, &supplied_token, candidate) - }) - .await?; - if operation.status == "success" { - return json_response(StatusCode::OK, &site_replication_repair_operation_response(&operation)); - } - - let service_account_secret_key = site_replicator_service_account_secret(&state.service_account_access_key).await?; - prepare_site_replication_repair_retry(&mut operation); - operation.status = "running".to_string(); - operation.completed_at = None; - operation.updated_at = Some(OffsetDateTime::now_utc()); - persist_site_replication_repair_operation(&operation).await?; - - let tasks = site_replication_repair_tasks(&plan); - for peer in state.peers.values().filter(|peer| { - peer.deployment_id != request.local_peer.deployment_id - && !same_identity_endpoint(&peer.endpoint, &request.local_peer.endpoint) - }) { - let transport = match PeerTransport::for_runtime_peer(peer).await { - Ok(transport) => transport, - Err(err) => { - let error = err.to_string(); - for (family_index, task) in &tasks { - if !site_replication_repair_task_pending(&operation, &peer.deployment_id, task.family(), *family_index)? { - continue; - } - update_site_replication_repair_task( - &mut operation, - &peer.deployment_id, - task.family(), - *family_index, - Err(&error), - )?; - summarize_site_replication_repair_operation(&mut operation); - persist_site_replication_repair_task(&operation, peer, task.family(), task.path()).await?; - } - continue; - } - }; - - for (family_index, task) in &tasks { - if !site_replication_repair_task_pending(&operation, &peer.deployment_id, task.family(), *family_index)? { - continue; - } - let result = task - .send(&transport, &state.service_account_access_key, &service_account_secret_key) - .await; - let error = result.err().map(|err| err.to_string()); - update_site_replication_repair_task( - &mut operation, - &peer.deployment_id, - task.family(), - *family_index, - match error.as_deref() { - Some(error) => Err(error), - None => Ok(()), - }, - )?; - summarize_site_replication_repair_operation(&mut operation); - persist_site_replication_repair_task(&operation, peer, task.family(), task.path()).await?; - } - } - - summarize_site_replication_repair_operation(&mut operation); - persist_site_replication_repair_operation(&operation).await?; - json_response(StatusCode::OK, &site_replication_repair_operation_response(&operation)) -} - -pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { - let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; - let runtime = { - // The bucket-op lock is what orders this against add/remove. The - // state is only read here (through the runtime snapshot), and the - // bucket setup below writes bucket metadata, never the state object — - // holding the state transaction across it would put local metadata - // IO inside a distributed lock for nothing. - let Some(runtime) = runtime_site_replication_targets().await? else { - return Ok(()); - }; - - ensure_site_replication_bucket_versioning(bucket).await?; - ensure_site_replication_bucket_setup_with_runtime(bucket, &runtime).await?; - runtime - }; - - broadcast_site_replication_make_bucket(bucket, lock_enabled, Some(&runtime), None).await -} - -async fn broadcast_site_replication_json_using_runtime( - runtime: Option<&SiteReplicationRuntime>, - path: &str, - body: &T, -) -> S3Result<()> { - match runtime { - Some(runtime) => broadcast_site_replication_json_with_runtime(runtime, path, body).await, - None => broadcast_site_replication_json(path, body).await, - } -} - -async fn broadcast_site_replication_make_bucket( - bucket: &str, - lock_enabled: bool, - runtime: Option<&SiteReplicationRuntime>, - bootstrap_token: Option<&str>, -) -> S3Result<()> { - let created_at = current_object_store_handle() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))? - .get_bucket_info(bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)? - .created - .unwrap_or_else(OffsetDateTime::now_utc) - .format(&time::format_description::well_known::Rfc3339) - .unwrap_or_default(); - - let path = make_with_versioning_bucket_op_path(bucket, Some(&created_at), lock_enabled); - let path = if let Some(token) = bootstrap_token { - with_site_replication_bootstrap_token(&path, token) - } else { - path - }; - broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await?; - - let configure_path = bootstrap_bucket_op_path(bucket, "configure-replication"); - let configure_path = if let Some(token) = bootstrap_token { - with_site_replication_bootstrap_token(&configure_path, token) - } else { - configure_path - }; - broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await -} - -pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> { - let operation = if force_delete { - "force-delete-bucket" - } else { - "delete-bucket" - }; - let path = format!( - "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", - form_urlencoded::Serializer::new(String::new()) - .append_pair("bucket", bucket) - .append_pair("operation", operation) - .finish() - ); - broadcast_site_replication_json(&path, &serde_json::json!({})).await -} - -pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> { - let Some(runtime) = runtime_site_replication_targets().await? else { - return Ok(()); - }; - if item.r#type == "lc-config" && !site_replication_state_replicates_ilm_expiry(&runtime.state) { - return Ok(()); - } - if item.r#type == "lc-config" { - // Only the expiry subset travels (MinIO peers install incoming rules - // verbatim, so transition rules must never leave this site). An empty - // subset becomes a delete, which the receiver merges with the empty - // set — local transition rules there survive. - item.expiry_lc_config = item - .expiry_lc_config - .and_then(|raw| lifecycle_expiry_subset_xml(raw.as_bytes())) - .map(|data| String::from_utf8_lossy(&data).into_owned()); - } - broadcast_site_replication_json_with_runtime( - &runtime, - "/rustfs/admin/v3/site-replication/peer/bucket-meta", - &encode_bucket_meta_wire_item(item), - ) - .await -} - -pub async fn site_replication_iam_change_hook(item: SRIAMItem) -> S3Result<()> { - broadcast_site_replication_json("/rustfs/admin/v3/site-replication/peer/iam-item", &item).await -} - -fn raw_config_to_string(raw: &[u8]) -> Option { - if raw.is_empty() { - return None; - } - String::from_utf8(raw.to_vec()).ok() -} - -fn raw_config_to_base64(raw: &[u8]) -> Option { - (!raw.is_empty()).then(|| BASE64_STANDARD.encode_to_string(raw)) -} - -fn encode_bucket_meta_wire_value(value: Option) -> Option { - value.map(|raw| BASE64_STANDARD.encode_to_string(raw.as_bytes())) -} - -fn encode_bucket_meta_wire_item(mut item: SRBucketMeta) -> SRBucketMeta { - item.versioning = encode_bucket_meta_wire_value(item.versioning); - item.tags = encode_bucket_meta_wire_value(item.tags); - item.object_lock_config = encode_bucket_meta_wire_value(item.object_lock_config); - item.sse_config = encode_bucket_meta_wire_value(item.sse_config); - item.replication_config = encode_bucket_meta_wire_value(item.replication_config); - item.expiry_lc_config = encode_bucket_meta_wire_value(item.expiry_lc_config); - item.cors = encode_bucket_meta_wire_value(item.cors); - item -} - -fn decode_bucket_meta_wire_value(raw: &str) -> Vec { - BASE64_STANDARD - .decode_to_vec(raw.as_bytes()) - .ok() - .filter(|decoded| std::str::from_utf8(decoded).is_ok()) - .unwrap_or_else(|| raw.as_bytes().to_vec()) -} - -fn decode_bucket_meta_wire_option(value: Option) -> Option> { - value.map(|raw| decode_bucket_meta_wire_value(&raw)) -} - -fn maybe_time(value: OffsetDateTime) -> Option { - (value != OffsetDateTime::UNIX_EPOCH).then_some(value) -} - -async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result { - let Some(store) = current_object_store_handle() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let mut info = SRInfo { - enabled: state.enabled(), - name: local_peer.name.clone(), - deployment_id: local_peer.deployment_id.clone(), - state: SRStateInfo { - name: local_peer.name.clone(), - peers: state.peers.clone(), - updated_at: state.updated_at, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }; - - let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?; - for bucket in buckets { - let metadata = metadata_sys::get(&bucket.name).await.ok(); - let mut entry = SRBucketInfo { - bucket: bucket.name.clone(), - created_at: bucket.created, - location: current_region().map(|region| region.to_string()).unwrap_or_default(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }; - - if let Some(metadata) = metadata { - entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); - entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml); - entry.tags = raw_config_to_base64(&metadata.tagging_config_xml); - entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml); - entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); - entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); - entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); - // Expiry subset only: this entry feeds both the bootstrap/repair - // plan (peers must not receive transition rules) and cross-site - // consistency views (transition rules are site-local and would - // read as false mismatches). A deleted expiry state is a `None` - // value with the deletion's axis so repair can converge peers - // that missed the live delete. - let expiry_statement = lifecycle_expiry_statement(&metadata); - entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); - entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); - entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); - entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); - entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); - entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); - entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); - entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); - entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); - // The expiry axis, not the whole-config write time: local - // transition-only edits inflate the latter, and a repair item - // stamped with it could out-rank a newer real expiry edit on a - // third site. - entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); - entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); - entry.replication_targets_online = - Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await); - } - - info.buckets.insert(bucket.name, entry); - } - - if let Some(iam_sys) = current_iam_handle() { - for (name, policy_doc) in iam_sys.list_policy_docs("").await.map_err(ApiError::from)? { - info.policies.insert( - name, - SRIAMPolicy { - policy: serde_json::to_value(policy_doc.policy).ok(), - updated_at: policy_doc.update_date, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }, - ); - } - - let users = iam_sys.list_users().await.map_err(ApiError::from)?; - for (name, user) in users { - info.user_info_map.insert(name, user); - } - - let groups = iam_sys.list_groups_load().await.map_err(ApiError::from)?; - for group in groups { - let desc = iam_sys.get_group_description(&group).await.map_err(ApiError::from)?; - info.group_desc_map.insert(group.clone(), desc); - } - - let mut user_policies = HashMap::::new(); - iam_sys - .load_mapped_policies(UserType::Reg, false, &mut user_policies) - .await - .map_err(ApiError::from)?; - for (name, mapping) in user_policies { - info.user_policies - .insert(name.clone(), mapped_policy_to_sr_mapping(name, false, UserType::Reg, mapping)); - } - - let mut group_policies = HashMap::::new(); - iam_sys - .load_mapped_policies(UserType::None, true, &mut group_policies) - .await - .map_err(ApiError::from)?; - for (name, mapping) in group_policies { - info.group_policies - .insert(name.clone(), mapped_policy_to_sr_mapping(name, true, UserType::None, mapping)); - } - } - - for (name, bucket_info) in &info.buckets { - if let Some(raw) = bucket_info - .replication_config - .as_ref() - .and_then(|value| serde_json::from_str::(value).ok()) - { - info.replication_cfg.insert(name.clone(), raw); - } - } - - Ok(info) -} - fn local_idp_settings() -> IDPSettings { let mut settings = IDPSettings::default(); if let Some(federation) = current_federated_identity_service() { @@ -4671,18 +1833,6 @@ fn local_idp_settings() -> IDPSettings { settings } -fn mapped_policy_to_sr_mapping(name: String, is_group: bool, user_type: UserType, mapping: MappedPolicy) -> SRPolicyMapping { - SRPolicyMapping { - user_or_group: name, - user_type: sr_wire_user_type(user_type, is_group), - is_group, - policy: mapping.policies, - updated_at: Some(mapping.update_at), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - } -} - fn filter_sr_info(mut info: SRInfo, opts: &SRStatusOptions) -> SRInfo { if opts.include_all_defaults() { return info; @@ -5535,16 +2685,6 @@ fn peer_endpoint_refresh_requested(state: &SiteReplicationState, incoming: &Peer .is_some_and(|peer| !peer_connection_settings_match(peer, incoming)) } -fn pending_endpoint_refresh(state: &SiteReplicationState) -> Option { - state.pending_endpoint_refresh.clone().or_else(|| { - state - .retry_queue - .iter() - .find(|event| event.path == SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH) - .and_then(|event| serde_json::from_str(&event.last_error).ok()) - }) -} - fn merge_pending_endpoint_refresh( state: &SiteReplicationState, candidate: &PendingEndpointRefresh, @@ -5632,41 +2772,10 @@ fn endpoint_refresh_target_state(state: &SiteReplicationState, pending: &Pending target_state } -fn parse_endpoint_refresh_status(peer: &PeerInfo, body: &[u8]) -> S3Result<()> { - let status: ReplicateEditStatus = serde_json::from_slice(body).map_err(|_| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("peer {} does not support endpoint target refresh", peer.endpoint), - ) - })?; - if status.success { - Ok(()) - } else { - Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("peer {} failed endpoint target refresh: {}", peer.endpoint, status.err_detail), - )) - } -} - fn endpoint_refresh_capability_supported(peer: &PeerInfo, status: StatusCode, body: &[u8]) -> S3Result { peer_capability_response_supported(peer, status, body) } -fn peer_capability_response_supported(peer: &PeerInfo, status: StatusCode, body: &[u8]) -> S3Result { - if status.is_success() { - return Ok(parse_endpoint_refresh_status(peer, body).is_ok()); - } - if matches!(status, StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED) { - return Ok(false); - } - - Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("probe site replication capability on peer {} failed with {status}", peer.endpoint), - )) -} - async fn require_add_peer_tls_capability(sites: &[PeerSite], local_peer: &PeerInfo) -> S3Result<()> { if !add_peer_tls_capability_required(sites) { return Ok(()); @@ -6167,87 +3276,6 @@ fn validate_remove_sites_req(state: &SiteReplicationState, req: &SRRemoveReq) -> Ok(()) } -fn summarize_peer_error_detail(detail: &str) -> String { - let detail = detail.trim(); - let detail_chars = detail.chars().count(); - if detail_chars <= SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT { - return detail.to_string(); - } - - let suffix = "... (truncated)"; - let take_chars = SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT.saturating_sub(suffix.chars().count()); - let mut summary: String = detail.chars().take(take_chars).collect(); - summary.push_str(suffix); - summary -} - -/// The wall clock in unix nanoseconds, clamped into u64. A pre-1970 (or -/// post-2554) clock yields 0, which makes the hybrid allocation below -/// degrade to the plain `previous + 1` counter — monotone, never panicking. -fn edit_generation_wall_clock() -> u64 { - u64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(0) -} - -/// Allocate the next peer-edit generation as a hybrid logical clock: -/// `max(wall clock in unix nanoseconds, previous + 1)`. Called inside the -/// state transaction, so the value is handed out under the distributed -/// state-object lock and two nodes of this site can never take the same one -/// (`previous + 1` keeps the sequence strictly increasing even when two -/// allocations land in one clock tick, and keeps it monotone on a node -/// whose clock stepped backwards mid-lifetime). -/// -/// The wall-clock floor is what survives the counter's death. A site -/// removed while unreachable — the receiver never dropped it from its peer -/// map, so the load-time mark pruning in `parse_site_replication_state` -/// never fired — that later rejoins recreates its state object with the -/// counter back at zero. A plain counter would then hand out generations -/// below the receiver's stale high-water mark and every delivery would be -/// silently fenced until the counter caught up. Jumping to wall time clears -/// that mark: every value the deleted lifetime handed out was capped by the -/// wall clock at its own allocation (or by a prior lifetime's cap, applied -/// inductively), so the recreated lifetime's first allocation exceeds them -/// all — while a pre-removal delivery still in flight stays below the new -/// floor and remains correctly fenced. Marks recorded by pre-hybrid -/// receivers (small plain-counter values) sit far below any wall-clock -/// value, so a restarted origin passes those too — the fix needs only the -/// sender upgraded, nothing on the wire or in the receiver changed. -/// -/// A wall clock that regresses across a delete/recreate (the recreating -/// node's clock behind the clock that fed the previous lifetime) mints -/// below the stale mark and the origin stays fenced — but only until real -/// time passes the previous lifetime's last allocation, because every later -/// allocation takes the wall-clock floor again (and never longer than -/// [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`]: a regression past the window -/// leaves the mark implausibly distant and the origin runs unfenced -/// immediately). Bounded by the skew, -/// self-healing, and no rollback window beyond the plain counter's: a -/// delivery applies only at or above the receiver's mark, so the one -/// cross-lifetime interleaving that can apply stale content — a -/// pre-removal delivery whose generation lands above everything the -/// regressed new lifetime has minted — required the same straggler landing -/// above the mark under the plain counter, where the recreated counter's -/// low restart made it strictly easier to hit. -fn next_peer_edit_generation(state: &mut SiteReplicationState) -> u64 { - state.edit_generation = edit_generation_wall_clock().max(state.edit_generation.saturating_add(1)); - state.edit_generation -} - -/// Build the peer-edit request path carrying the fencing token. The bare -/// constant stays the retry-queue key: the query only fences the wire -/// delivery, and a per-generation key would make every retry event unique. -/// Without a local deployment id there is nothing to fence against, so the -/// unstamped path is sent and the receiver keeps its pre-fence behaviour. -fn peer_edit_path_with_fence(origin: Option<&str>, generation: u64) -> String { - let Some(origin) = origin.filter(|origin| !origin.is_empty()) else { - return SITE_REPLICATION_PEER_EDIT_PATH.to_string(); - }; - let query = form_urlencoded::Serializer::new(String::new()) - .append_pair(SITE_REPLICATION_EDIT_ORIGIN_QUERY, origin) - .append_pair(SITE_REPLICATION_EDIT_GENERATION_QUERY, &generation.to_string()) - .finish(); - format!("{SITE_REPLICATION_PEER_EDIT_PATH}?{query}") -} - /// The (origin site, generation) fence an incoming peer edit carries, when the /// sender stamped one. An unstamped edit (older peer) has no fence and is /// applied as before. @@ -6355,881 +3383,6 @@ fn record_applied_peer_edit_generation(state: &mut SiteReplicationState, origin: *applied = (*applied).max(generation); } -fn retry_event_matches(event: &SiteReplicationRetryEvent, peer: &PeerInfo, path: &str) -> bool { - (event.peer_deployment_id == peer.deployment_id || event.peer_endpoint == peer.endpoint) && event.path == path -} - -const SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH: &str = "internal:retry-snapshot:iam"; -const SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH: &str = "internal:retry-snapshot:bucket-metadata"; - -fn collapsed_retry_queue_path(path: &str) -> Option<&'static str> { - let base_path = path.split_once('?').map(|(base, _)| base).unwrap_or(path); - match base_path { - "/rustfs/admin/v3/site-replication/peer/iam-item" | SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => { - Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) - } - "/rustfs/admin/v3/site-replication/peer/bucket-meta" | SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => { - Some(SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH) - } - _ => None, - } -} - -fn normalize_collapsed_retry_queue_paths(queue: &mut Vec) -> bool { - let mut changed = false; - let mut normalized: Vec = Vec::with_capacity(queue.len()); - for mut event in queue.drain(..) { - if let Some(path) = collapsed_retry_queue_path(&event.path) - && event.path != path - { - event.path = path.to_string(); - changed = true; - } - - let duplicate = normalized.iter().position(|existing| { - existing.path == event.path - && (existing.peer_deployment_id == event.peer_deployment_id || existing.peer_endpoint == event.peer_endpoint) - }); - let Some(index) = duplicate else { - normalized.push(event); - continue; - }; - - changed = true; - let existing = &mut normalized[index]; - let event_is_newer = match (event.updated_at, existing.updated_at) { - (Some(event), Some(existing)) => event >= existing, - (Some(_), None) => true, - _ => false, - }; - if event_is_newer { - let retry_count = existing.retry_count.max(event.retry_count); - *existing = event; - existing.retry_count = retry_count; - } else { - existing.retry_count = existing.retry_count.max(event.retry_count); - } - existing.failed = existing.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; - } - *queue = normalized; - changed -} - -async fn migrate_collapsed_retry_queue_paths() -> S3Result<()> { - update_site_replication_state_when_changed(|state| { - Ok(if normalize_collapsed_retry_queue_paths(&mut state.retry_queue) { - StateCommit::Changed(()) - } else { - StateCommit::Unchanged(()) - }) - }) - .await -} - -#[cfg(test)] -fn dequeue_site_replication_retry_events(queue: &mut Vec, peer: &PeerInfo, path: &str) -> usize { - settle_site_replication_retry_events(queue, peer, path, None) -} - -/// Repair-path settlement: also clears snapshot-escalated entries. Running a -/// repair is the operator's explicit accountability transfer for the -/// possibly-unreplayed deletion the marker records; ordinary delivery -/// successes must not clear it (see [`settle_site_replication_retry_events`]). -fn dequeue_site_replication_retry_events_including_escalated( - queue: &mut Vec, - peer: &PeerInfo, - path: &str, -) -> usize { - let before = queue.len(); - let collapsed_path = collapsed_retry_queue_path(path); - queue.retain(|event| { - !retry_event_matches(event, peer, path) - && !collapsed_path.is_some_and(|collapsed_path| retry_event_matches(event, peer, collapsed_path)) - }); - before.saturating_sub(queue.len()) -} - -/// Remove the retry events for (peer, path) that `generation` is entitled to -/// settle. A successful delivery only proves the peer reached the state the -/// delivery carried: while it was in flight another edit can commit, fail its -/// own delivery, and enqueue for the same (peer, path). Erasing that event -/// would leave the peer on the older edit with no retry left, so an event -/// stamped with a NEWER generation survives. `None` settles unconditionally — -/// the broadcast paths that carry no generation, whose retry events live under -/// their own paths and never collide with peer-edit deliveries. -fn settle_site_replication_retry_events( - queue: &mut Vec, - peer: &PeerInfo, - path: &str, - generation: Option, -) -> usize { - let before = queue.len(); - let collapsed_path = collapsed_retry_queue_path(path); - queue.retain(|event| { - if !retry_event_matches(event, peer, path) { - return true; - } - // A wire-path success identifies no IAM or bucket-metadata entity. - // This also protects legacy rows until the startup migration moves - // them under their internal snapshot path. - if collapsed_path.is_some() { - return true; - } - // A snapshot-escalated entry records a possibly-unreplayed deletion. - // Collapsed paths are shared by every entity, so a later successful - // delivery of a DIFFERENT item proves nothing about the deleted one — - // only a repair settles it (dequeue_..._including_escalated). - if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { - return true; - } - match (generation, event.edit_generation) { - (Some(settled), Some(failed)) => failed > settled, - _ => false, - } - }); - before.saturating_sub(queue.len()) -} - -fn upsert_site_replication_retry_event( - queue: &mut Vec, - peer: &PeerInfo, - path: &str, - error: &str, - generation: Option, -) { - let path = collapsed_retry_queue_path(path).unwrap_or(path); - let now = OffsetDateTime::now_utc(); - let detail = summarize_peer_error_detail(error); - if let Some(event) = queue.iter_mut().find(|event| retry_event_matches(event, peer, path)) { - event.retry_count = event.retry_count.saturating_add(1); - event.failed = event.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; - event.last_error = detail; - event.updated_at = Some(now); - // Keep the newest generation: an older delivery that fails afterwards - // must not lower the fence and let its own success settle the event. - event.edit_generation = event.edit_generation.max(generation); - return; - } - - queue.push(SiteReplicationRetryEvent { - id: Uuid::new_v4().to_string(), - peer_deployment_id: peer.deployment_id.clone(), - peer_endpoint: peer.endpoint.clone(), - path: path.to_string(), - retry_count: 1, - failed: false, - last_error: detail, - updated_at: Some(now), - edit_generation: generation, - }); - if queue.len() > SITE_REPLICATION_RETRY_QUEUE_LIMIT { - let overflow = queue.len() - SITE_REPLICATION_RETRY_QUEUE_LIMIT; - queue.drain(0..overflow); - } -} - -fn retry_stats_for_state(state: &SiteReplicationState) -> Option { - if state.retry_queue.is_empty() { - return None; - } - - Some(SRRetryStats { - pending: state.retry_queue.iter().filter(|event| !event.failed).count(), - failed: state.retry_queue.iter().filter(|event| event.failed).count(), - last_error: state - .retry_queue - .iter() - .rev() - .find_map(|event| (!event.last_error.is_empty()).then(|| event.last_error.clone())) - .unwrap_or_default(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }) -} - -async fn enqueue_site_replication_retry_event(peer: &PeerInfo, path: &str, error: &S3Error) { - enqueue_site_replication_retry_event_for_generation(peer, path, error, None).await -} - -async fn enqueue_site_replication_retry_event_for_generation( - peer: &PeerInfo, - path: &str, - error: &S3Error, - generation: Option, -) { - let peer_owned = peer.clone(); - let path_owned = path.to_string(); - let error_text = error.to_string(); - let result = update_site_replication_state(move |state| { - // A peer that left the state can never drain its entries again - // (remove_sites already pruned them); recording a late failure for it - // would only pollute retry_stats until the queue cap evicts it. - if state.peers.contains_key(&peer_owned.deployment_id) { - upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); - } - Ok(()) - }) - .await; - - if let Err(err) = result { - warn!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - peer = %peer.endpoint, - path, - error = ?err, - "failed to persist site replication retry event" - ); - } -} - -fn retry_bucket_operation(path: &str) -> Option { - let (base_path, query) = path.split_once('?')?; - if base_path != SITE_REPLICATION_PEER_BUCKET_OPS_PATH { - return None; - } - - form_urlencoded::parse(query.as_bytes()).find_map(|(key, value)| (key == "operation").then(|| value.into_owned())) -} - -fn retry_event_replayed_by_bootstrap(event: &SiteReplicationRetryEvent) -> bool { - matches!( - retry_bucket_operation(&event.path).as_deref(), - Some(SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING | SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION) - ) -} - -/// Exponential backoff base for the background retry drain, aligned with the -/// reconcile cadence (`site_replication_reconcile::RECONCILE_INTERVAL`). -const SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS: i64 = 600; -/// Backoff ceiling: a permanently failed peer is still probed daily. -const SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS: i64 = 86_400; - -/// What the background drain may do for one retry event. Everything not -/// representable here is operator territory (manual repair). -#[derive(Debug, Clone, PartialEq, Eq)] -enum RetryDrainAction { - /// Constant-path IAM item deliveries collapse into one queue entry per - /// peer and their bodies are not persisted; the only faithful replay is - /// the current IAM snapshot from the bootstrap plan. - IamSnapshot, - /// Same collapse for bucket-meta deliveries: replay the bucket metadata - /// snapshot from the bootstrap plan. - BucketMetadataSnapshot, - /// A self-contained bucket op the bootstrap plan can re-derive for its - /// bucket (`make-with-versioning` / `configure-replication`). - BucketOpReplay { operation: String, bucket: String }, - /// Re-send the current peer records under a fresh edit generation. - PeerEdit, -} - -#[derive(Clone)] -enum RetrySnapshot { - Iam(Vec), - BucketMetadata(Vec), -} - -impl RetrySnapshot { - fn from_plan(action: &RetryDrainAction, plan: &SiteReplicationBootstrapPlan) -> Option { - match action { - RetryDrainAction::IamSnapshot => Some(Self::Iam(plan.iam_items.clone())), - RetryDrainAction::BucketMetadataSnapshot => Some(Self::BucketMetadata(plan.bucket_items.clone())), - _ => None, - } - } - - fn fingerprint(&self) -> S3Result>> { - let mut payloads = match self { - Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), - Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), - } - .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?; - payloads.sort_unstable(); - Ok(payloads) - } - - fn replay_after_change(previous: &Self, fresh: &Self, observed_at: OffsetDateTime) -> Self { - match (previous, fresh) { - (Self::Iam(previous), Self::Iam(fresh)) => { - let fresh_keys: HashSet = fresh.iter().filter_map(iam_snapshot_key).collect(); - let mut replay = fresh.clone(); - for item in previous { - if iam_snapshot_key(item).is_some_and(|key| !fresh_keys.contains(&key)) { - replay.extend(iam_snapshot_tombstones(item, observed_at)); - } - } - Self::Iam(replay) - } - (Self::BucketMetadata(previous), Self::BucketMetadata(fresh)) => { - let fresh_keys: HashSet<(&str, &str)> = fresh - .iter() - .map(|item| (item.bucket.as_str(), item.r#type.as_str())) - .collect(); - let mut replay = fresh.clone(); - for item in previous { - if !fresh_keys.contains(&(item.bucket.as_str(), item.r#type.as_str())) { - replay.push(bucket_metadata_snapshot_tombstone(item, observed_at)); - } - } - Self::BucketMetadata(replay) - } - _ => fresh.clone(), - } - } - - async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<()> { - match self { - Self::Iam(items) => { - for item in items { - SiteReplicationRepairTask::Iam(item) - .send(transport, access_key, secret_key) - .await?; - } - } - Self::BucketMetadata(items) => { - for item in items { - SiteReplicationRepairTask::BucketMetadata(item) - .send(transport, access_key, secret_key) - .await?; - } - } - } - Ok(()) - } -} - -#[derive(Hash, PartialEq, Eq)] -enum IamSnapshotKey { - Policy(String), - User(String), - Group(String), - PolicyMapping { target: String, user_type: i64, is_group: bool }, -} - -fn iam_snapshot_key(item: &SRIAMItem) -> Option { - match item.r#type.as_str() { - "policy" => Some(IamSnapshotKey::Policy(item.name.clone())), - "iam-user" => item - .iam_user - .as_ref() - .map(|user| IamSnapshotKey::User(user.access_key.clone())), - "group-info" => item - .group_info - .as_ref() - .map(|group| IamSnapshotKey::Group(group.update_req.group.clone())), - "policy-mapping" => item.policy_mapping.as_ref().map(|mapping| IamSnapshotKey::PolicyMapping { - target: mapping.user_or_group.clone(), - user_type: mapping.user_type, - is_group: mapping.is_group, - }), - _ => None, - } -} - -fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateTime) -> Vec { - let mut tombstone = item.clone(); - tombstone.updated_at = Some(observed_at); - match item.r#type.as_str() { - "policy" => tombstone.policy = None, - "iam-user" => { - if let Some(user) = tombstone.iam_user.as_mut() { - user.is_delete_req = true; - user.user_req = None; - } - } - "group-info" => { - let Some(group) = tombstone.group_info.as_mut() else { - return Vec::new(); - }; - group.update_req.is_remove = true; - if group.update_req.members.is_empty() { - return vec![tombstone]; - } - let mut delete = tombstone.clone(); - if let Some(group) = delete.group_info.as_mut() { - group.update_req.members.clear(); - } - return vec![tombstone, delete]; - } - "policy-mapping" => { - if let Some(mapping) = tombstone.policy_mapping.as_mut() { - mapping.policy.clear(); - } - } - _ => return Vec::new(), - } - vec![tombstone] -} - -fn bucket_metadata_snapshot_tombstone(item: &SRBucketMeta, observed_at: OffsetDateTime) -> SRBucketMeta { - SRBucketMeta { - r#type: item.r#type.clone(), - bucket: item.bucket.clone(), - updated_at: Some(observed_at), - expiry_updated_at: Some(observed_at), - api_version: item.api_version.clone(), - derived_rule_contract: item.derived_rule_contract, - ..Default::default() - } -} - -const SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS: usize = 3; - -fn classify_site_replication_retry_event(event: &SiteReplicationRetryEvent) -> Option { - let snapshot_action = match event.path.as_str() { - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => Some(RetryDrainAction::IamSnapshot), - SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => Some(RetryDrainAction::BucketMetadataSnapshot), - _ => None, - }; - if snapshot_action.is_some() && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { - return snapshot_action; - } - if event.path.starts_with("internal:") { - // Marker records store payloads in `last_error` (legacy - // pending-endpoint-refresh backup and snapshot liabilities); they are - // not drainable delivery failures. - return None; - } - if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { - // Already snapshot-replayed once for this failure episode; a possible - // deletion cannot be replayed from a snapshot, so re-sending daily - // proves nothing. A new hook failure overwrites the marker. - return None; - } - let base_path = event.path.split_once('?').map(|(base, _)| base).unwrap_or(&event.path); - match base_path { - "/rustfs/admin/v3/site-replication/peer/iam-item" => Some(RetryDrainAction::IamSnapshot), - "/rustfs/admin/v3/site-replication/peer/bucket-meta" => Some(RetryDrainAction::BucketMetadataSnapshot), - SITE_REPLICATION_PEER_EDIT_PATH => Some(RetryDrainAction::PeerEdit), - SITE_REPLICATION_PEER_BUCKET_OPS_PATH => { - let operation = retry_bucket_operation(&event.path)?; - if !matches!( - operation.as_str(), - SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING | SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION - ) { - // Destructive ops (delete-bucket / force-delete-bucket) are - // operator territory: replaying them against a peer whose - // bucket was since recreated is irreversible. - return None; - } - let bucket = retry_bucket_name(&event.path)?; - Some(RetryDrainAction::BucketOpReplay { operation, bucket }) - } - _ => None, - } -} - -fn retry_bucket_name(path: &str) -> Option { - let (_, query) = path.split_once('?')?; - form_urlencoded::parse(query.as_bytes()) - .find_map(|(key, value)| (key == "bucket" && !value.is_empty()).then(|| value.into_owned())) -} - -/// A collapsed retry event after a stable snapshot resend is escalated with -/// this marker instead of being cleared: the snapshot contains no task for a -/// failed deletion, so remote absence remains operator-visible. Collapsed -/// failures use an internal queue path so ordinary successes and older nodes -/// cannot settle an unrelated entity's liability. -const SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER: &str = "snapshot replayed; a failed deletion cannot be replayed from a snapshot — run site replication repair or re-deliver to settle"; - -/// Escalate a collapsed retry event after its snapshot resend succeeded, -/// unless a newer failure was recorded after `snapshot_updated_at` (that -/// failure belongs to a newer local commit the snapshot did not contain and -/// must keep the entry drain-eligible). -fn escalate_site_replication_retry_events_up_to( - queue: &mut Vec, - peer: &PeerInfo, - path: &str, - snapshot_updated_at: Option, -) -> usize { - let Some(marker_path) = collapsed_retry_queue_path(path) else { - return 0; - }; - - if path != marker_path { - queue.retain(|event| { - if !retry_event_matches(event, peer, path) { - return true; - } - matches!((event.updated_at, snapshot_updated_at), (Some(current), Some(seen)) if current > seen) - || matches!((event.updated_at, snapshot_updated_at), (Some(_), None)) - }); - } - - let marker_index = queue.iter().position(|event| retry_event_matches(event, peer, marker_path)); - let marker_index = marker_index.unwrap_or_else(|| { - queue.push(SiteReplicationRetryEvent { - id: Uuid::new_v4().to_string(), - peer_deployment_id: peer.deployment_id.clone(), - peer_endpoint: peer.endpoint.clone(), - path: marker_path.to_string(), - updated_at: snapshot_updated_at, - ..Default::default() - }); - queue.len() - 1 - }); - let event = &mut queue[marker_index]; - let newer_failure_recorded = match (event.updated_at, snapshot_updated_at) { - (Some(current), Some(seen)) => current > seen, - (Some(_), None) => true, - (None, _) => false, - }; - if newer_failure_recorded && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { - return 0; - } - event.failed = true; - event.retry_count = event.retry_count.max(SITE_REPLICATION_RETRY_FAILED_AFTER); - event.last_error = SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER.to_string(); - event.updated_at = Some(OffsetDateTime::now_utc()); - 1 -} - -async fn escalate_site_replication_retry_event_up_to(peer: &PeerInfo, path: &str, snapshot_updated_at: Option) { - let peer_owned = peer.clone(); - let path_owned = path.to_string(); - let result = update_site_replication_state(move |state| { - escalate_site_replication_retry_events_up_to(&mut state.retry_queue, &peer_owned, &path_owned, snapshot_updated_at); - Ok(()) - }) - .await; - - if let Err(err) = result { - warn!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - peer = %peer.endpoint, - deployment_id = %peer.deployment_id, - path, - error = ?err, - "failed to escalate site replication retry event" - ); - } -} - -/// Whether the drain may attempt this event now. -fn site_replication_retry_backoff_elapsed(event: &SiteReplicationRetryEvent, now: OffsetDateTime) -> bool { - let Some(updated_at) = event.updated_at else { - return true; - }; - // 600 * 2^8 already exceeds the daily ceiling; capping the shift keeps - // the arithmetic overflow-free for any persisted retry_count. - let exponent = event.retry_count.saturating_sub(1).min(8); - let delay = (SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS << exponent).min(SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS); - now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= delay -} - -/// The subset of the retry queue the background drain is allowed to touch. -fn actionable_site_replication_retry_events(state: &SiteReplicationState, now: OffsetDateTime) -> Vec { - state - .retry_queue - .iter() - .filter(|event| classify_site_replication_retry_event(event).is_some()) - .filter(|event| state.peers.contains_key(&event.peer_deployment_id)) - .filter(|event| site_replication_retry_backoff_elapsed(event, now)) - .cloned() - .collect() -} - -/// Background consumer for the retry queue, run from the reconcile tick. -/// -/// Scope: this settles "delivered once and failed" entries whose replay is -/// faithful (bucket ops, peer edits). Collapsed iam-item / bucket-meta -/// entries are snapshot-resent and then *escalated*, not cleared — a failed -/// deletion leaves no task in the snapshot, so remote absence stays unproven -/// until a later delivery or a manual repair. A hook that never fired (crash -/// between the local commit and the send) leaves no entry at all, so the -/// drain is not a full cross-site diff-heal; manual repair remains the -/// authoritative catch-all. -async fn drain_site_replication_retry_queue() { - if let Err(err) = drain_site_replication_retry_queue_inner().await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - result = "retry_drain_failed", - error = ?err, - "admin site replication state" - ); - } -} - -async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { - let Some(runtime) = runtime_site_replication_targets().await? else { - return Ok(()); - }; - let actionable = actionable_site_replication_retry_events(&runtime.state, OffsetDateTime::now_utc()); - if actionable.is_empty() { - return Ok(()); - } - let Some(store) = current_object_store_handle() else { - return Ok(()); - }; - if runtime.state.pending_endpoint_refresh.is_some() - || runtime.state.pending_remove.is_some() - || runtime.state.pending_rotation.is_some() - { - // The tick-level gate ran before the reconcilers; a multi-step flow - // (endpoint refresh commits its pending marker without the lifecycle - // guard) may have started since. Re-check on the fresh state. - return Ok(()); - } - // Serialize against operator repair execution. This does NOT close the - // dry-run -> execute window (dry-run takes no lock): a drain settling a - // replayable bucket-op entry in that window changes the preflight token - // and execute fails safe with "preflight is stale" — the operator - // re-runs the dry-run. Lock order matches repair: lifecycle guard (held - // by the reconcile tick) -> repair execution lock -> state object lock - // inside the send bookkeeping. An operator repair holding the lock makes - // this tick skip after the lock-acquire timeout. - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { - drain_site_replication_retry_queue_locked(runtime, actionable).await - }) - .await - .map_err(ApiError::from)? -} - -async fn drain_site_replication_retry_queue_locked( - runtime: SiteReplicationRuntime, - events: Vec, -) -> S3Result<()> { - let needs_plan = events - .iter() - .any(|event| !matches!(classify_site_replication_retry_event(event), Some(RetryDrainAction::PeerEdit))); - // The plan is a full local snapshot (buckets + IAM); build it once per - // tick and only when a snapshot resend is actually due. - let plan = if needs_plan { - let info = build_sr_info(&runtime.state, &runtime.local_peer).await?; - Some(site_replication_bootstrap_plan(&info)?) - } else { - None - }; - - let mut events_by_peer: BTreeMap> = BTreeMap::new(); - for event in events { - events_by_peer - .entry(event.peer_deployment_id.clone()) - .or_default() - .push(event); - } - - let mut settled = 0usize; - let mut failures = 0usize; - for (deployment_id, peer_events) in events_by_peer { - let Some(peer) = runtime.state.peers.get(&deployment_id) else { - continue; - }; - if deployment_id == runtime.local_peer.deployment_id - || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) - { - continue; - } - let transport = match PeerTransport::for_runtime_peer(peer).await { - Ok(transport) => transport, - Err(err) => { - // Record the attempt so backoff advances for an unreachable - // peer instead of re-dialing it every tick. - for event in &peer_events { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - } - failures += peer_events.len(); - continue; - } - }; - for event in peer_events { - let Some(action) = classify_site_replication_retry_event(&event) else { - continue; - }; - match drain_one_site_replication_retry_event(&runtime, peer, &transport, &event, action, plan.as_ref()).await { - Ok(true) => settled += 1, - Ok(false) => {} - Err(_) => failures += 1, - } - } - } - - if settled > 0 || failures > 0 { - info!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - result = "retry_drain_settled", - settled, - failures, - "admin site replication state" - ); - } - Ok(()) -} - -/// Replay one retry event against its peer. Returns `Ok(true)` when the -/// event was settled (delivered, or provably stale), `Ok(false)` when it was -/// skipped, and `Err` after a failed delivery (already re-queued with an -/// incremented retry count). -async fn drain_one_site_replication_retry_event( - runtime: &SiteReplicationRuntime, - peer: &PeerInfo, - transport: &PeerTransport, - event: &SiteReplicationRetryEvent, - action: RetryDrainAction, - plan: Option<&SiteReplicationBootstrapPlan>, -) -> S3Result { - let access_key = &runtime.state.service_account_access_key; - let secret_key = &runtime.service_account_secret_key; - match action.clone() { - RetryDrainAction::IamSnapshot | RetryDrainAction::BucketMetadataSnapshot => { - let Some(plan) = plan else { - return Ok(false); - }; - let mut current_snapshot = RetrySnapshot::from_plan(&action, plan).expect("snapshot action has a snapshot"); - let mut replay = current_snapshot.clone(); - for _ in 0..SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS { - let current_fingerprint = current_snapshot.fingerprint()?; - if let Err(err) = replay.send(transport, access_key, secret_key).await { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - return Err(err); - } - let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; - let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; - let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); - if fresh_snapshot.fingerprint()? == current_fingerprint { - escalate_site_replication_retry_event_up_to(peer, &event.path, event.updated_at).await; - return Ok(true); - } - replay = RetrySnapshot::replay_after_change(¤t_snapshot, &fresh_snapshot, OffsetDateTime::now_utc()); - current_snapshot = fresh_snapshot; - } - Ok(false) - } - RetryDrainAction::BucketOpReplay { operation, bucket } => { - let Some(plan) = plan else { - return Ok(false); - }; - // Replay from the CURRENT plan, never the recorded path: the - // recorded query can carry an expired one-shot bootstrap token or - // a stale createdAt. - let make_op = operation == SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING; - let paths = if make_op { - &plan.bucket_make_ops - } else { - &plan.bucket_configure_ops - }; - let tasks: Vec> = paths - .iter() - .filter(|path| retry_bucket_name(path).as_deref() == Some(bucket.as_str())) - .map(|path| { - if make_op { - SiteReplicationRepairTask::BucketMake(path) - } else { - SiteReplicationRepairTask::Replication(path) - } - }) - .collect(); - if tasks.is_empty() { - // The bucket left the plan (deleted, or replication no longer - // configured): the recorded intent is stale, settle it. - dequeue_site_replication_retry_event(peer, &event.path).await; - return Ok(true); - } - for task in &tasks { - if let Err(err) = task.send(transport, access_key, secret_key).await { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - return Err(err); - } - } - dequeue_site_replication_retry_event(peer, &event.path).await; - Ok(true) - } - RetryDrainAction::PeerEdit => { - // The recorded generation is stale by definition — the receiver - // fences it. Allocate a fresh generation and re-send the current - // peer records (a superset of the failed body; the receiver - // upserts), all inside one state transaction so the fence and the - // bodies agree. - let target_id = peer.deployment_id.clone(); - let (generation, bodies) = update_site_replication_state(move |state| { - if !state.peers.contains_key(&target_id) { - return Ok((None, Vec::new())); - } - Ok((Some(next_peer_edit_generation(state)), state.peers.values().cloned().collect::>())) - }) - .await?; - let Some(generation) = generation else { - // Peer left between the snapshot and now; the queue entry was - // already pruned by remove_sites. - return Ok(false); - }; - let local_deployment_id = Some(runtime.local_peer.deployment_id.as_str()).filter(|id| !id.is_empty()); - let edit_path = peer_edit_path_with_fence(local_deployment_id, generation); - let delivery_fence = local_deployment_id.is_some().then_some(generation); - for body in &bodies { - if let Err(err) = send_peer_admin_request_with_client( - &transport.client, - &transport.connection, - &edit_path, - access_key, - secret_key, - body, - ) - .await - { - enqueue_site_replication_retry_event_for_generation( - peer, - SITE_REPLICATION_PEER_EDIT_PATH, - &err, - delivery_fence, - ) - .await; - return Err(err); - } - } - dequeue_site_replication_retry_event_for_generation(peer, SITE_REPLICATION_PEER_EDIT_PATH, delivery_fence).await; - Ok(true) - } - } -} - -/// Remove a retry event for (peer, path) from the queue on successful delivery. -/// This is a no-op (load + no-op persist skipped) when no matching entry exists, -/// avoiding unnecessary I/O on the common path. -async fn dequeue_site_replication_retry_event(peer: &PeerInfo, path: &str) { - dequeue_site_replication_retry_event_for_generation(peer, path, None).await -} - -async fn dequeue_site_replication_retry_event_for_generation(peer: &PeerInfo, path: &str, generation: Option) { - let result = async { - // Fast path: this sits on every successful hook broadcast, so probe - // with a plain read first and only enter the locked RMW on a hit - // (the transaction re-checks under the lock). - let mut probe = load_site_replication_state().await?; - if settle_site_replication_retry_events(&mut probe.retry_queue, peer, path, generation) == 0 { - return Ok(()); - } - let peer_owned = peer.clone(); - let path_owned = path.to_string(); - update_site_replication_state(move |state| { - settle_site_replication_retry_events(&mut state.retry_queue, &peer_owned, &path_owned, generation); - Ok(()) - }) - .await?; - Ok::<_, S3Error>(()) - } - .await; - - if let Err(err) = result { - warn!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - peer = %peer.endpoint, - deployment_id = %peer.deployment_id, - path, - error = ?err, - "failed to dequeue site replication retry event" - ); - } -} - /// The removal's client-facing verdict. /// /// A fully-notified removal keeps answering with the historical success string, @@ -7737,161 +3890,6 @@ fn parse_site_resync_page(query: &HashMap, status: &SRResyncOpSt Ok((limit, offset)) } -fn bucket_target_endpoint(target: &BucketTarget) -> String { - let scheme = if target.secure { "https" } else { "http" }; - canonical_endpoint(&format!("{scheme}://{}", target.endpoint)) -} - -fn bucket_target_matches_peer(target: &BucketTarget, peer: &PeerInfo) -> bool { - if !target.deployment_id.is_empty() { - return target.deployment_id == peer.deployment_id; - } - bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint) -} - -fn site_replication_target_arns_by_peer(config: Option<&s3s::dto::ReplicationConfiguration>) -> HashMap { - let mut arns_by_peer = HashMap::new(); - let Some(config) = config else { - return arns_by_peer; - }; - - let mut configured_arns = Vec::new(); - if !config.role.trim().is_empty() { - configured_arns.push(config.role.clone()); - } - for rule in &config.rules { - let arn = rule.destination.bucket.trim(); - if !arn.is_empty() { - configured_arns.push(arn.to_string()); - } - } - - for arn in configured_arns { - if let Some(deployment_id) = replication_target_arn_deployment_id(&arn) { - arns_by_peer.entry(deployment_id).or_insert(arn); - } - } - - arns_by_peer -} - -fn site_replication_bucket_target_for_peer( - bucket: &str, - state: &SiteReplicationState, - peer: &PeerInfo, - service_account_secret_key: &str, - arn_override: Option, -) -> S3Result> { - if state.service_account_access_key.is_empty() || service_account_secret_key.is_empty() { - return Ok(None); - } - - let parsed = Url::parse(&peer.endpoint) - .ok() - .or_else(|| Url::parse(&format!("http://{}", peer.endpoint.trim())).ok()) - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {}", peer.endpoint)))?; - let host = parsed.host_str().ok_or_else(|| { - S3Error::with_message(S3ErrorCode::InvalidRequest, format!("peer endpoint missing host: {}", peer.endpoint)) - })?; - let port = parsed.port_or_known_default().ok_or_else(|| { - S3Error::with_message(S3ErrorCode::InvalidRequest, format!("peer endpoint missing port: {}", peer.endpoint)) - })?; - let region = current_region() - .map(|region| region.to_string()) - .filter(|region| !region.is_empty()) - .unwrap_or_else(|| "us-east-1".to_string()); - let arn = arn_override.unwrap_or_else(|| { - ARN::new( - BucketTargetType::ReplicationService, - peer.deployment_id.clone(), - String::new(), - bucket.to_string(), - ) - .to_string() - }); - - Ok(Some(BucketTarget { - source_bucket: bucket.to_string(), - endpoint: format!("{host}:{port}"), - credentials: Some(Credentials { - access_key: state.service_account_access_key.clone(), - secret_key: service_account_secret_key.to_string(), - session_token: None, - expiration: None, - }), - target_bucket: bucket.to_string(), - secure: parsed.scheme().eq_ignore_ascii_case("https"), - arn, - region, - target_type: BucketTargetType::ReplicationService, - deployment_id: peer.deployment_id.clone(), - skip_tls_verify: peer.skip_tls_verify, - ca_cert_pem: peer.ca_cert_pem.clone(), - ..Default::default() - })) -} - -fn reconcile_site_replication_bucket_targets( - existing: BucketTargets, - bucket: &str, - state: &SiteReplicationState, - local_peer: &PeerInfo, - config: Option<&s3s::dto::ReplicationConfiguration>, - service_account_secret_key: &str, -) -> S3Result { - if !state.enabled() || state.service_account_access_key.is_empty() || service_account_secret_key.is_empty() { - return Ok(existing); - } - - let configured_arns = site_replication_target_arns_by_peer(config); - let mut targets = existing.targets; - - for peer in state.peers.values() { - if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { - continue; - } - - let Some(mut target) = site_replication_bucket_target_for_peer( - bucket, - state, - peer, - service_account_secret_key, - configured_arns.get(&peer.deployment_id).cloned(), - )? - else { - continue; - }; - - if let Some(index) = targets.iter().position(|existing| { - existing.target_type == BucketTargetType::ReplicationService - && (bucket_target_matches_peer(existing, peer) || existing.arn == target.arn) - }) { - let existing = targets[index].clone(); - target.path = existing.path; - target.region = existing.region; - target.bandwidth_limit = existing.bandwidth_limit; - target.replication_sync = existing.replication_sync; - target.storage_class = existing.storage_class; - target.health_check_duration = existing.health_check_duration; - target.disable_proxy = existing.disable_proxy; - target.reset_before_date = existing.reset_before_date; - target.reset_id = existing.reset_id; - target.total_downtime = existing.total_downtime; - target.last_online = existing.last_online; - target.online = existing.online; - target.latency = existing.latency; - target.edge = existing.edge; - target.edge_sync_before_expiry = existing.edge_sync_before_expiry; - target.offline_count = existing.offline_count; - targets[index] = target; - } else { - targets.push(target); - } - } - - Ok(BucketTargets { targets }) -} - fn bucket_target_deployment_id(target: &BucketTarget) -> Option { if !target.deployment_id.trim().is_empty() { return Some(target.deployment_id.clone()); @@ -7923,31 +3921,6 @@ fn prune_removed_site_replication_bucket_targets( (BucketTargets { targets }, removed) } -/// Whether every `site-repl-*` rule on this bucket resolves to a live remote target. -/// -/// The rule set alone cannot answer this: a rule can be perfectly formed while the endpoint -/// recorded for its peer is one this site cannot reach, so `update_all_targets` never built -/// a client for it and `replicate_object` drops every object against that ARN. Reads the -/// already-resolved client map rather than rebuilding clients, so it stays cheap enough for -/// the status path. -async fn site_replication_targets_online(bucket: &str, replication_config_xml: &[u8]) -> bool { - let Ok(config) = deserialize::(replication_config_xml) else { - return true; - }; - - for rule in config.rules.iter().filter(|rule| is_derived_site_replication_rule(rule)) { - if BucketTargetSys::get() - .get_remote_target_client_by_arn(bucket, &rule.destination.bucket) - .await - .is_none() - { - return false; - } - } - - true -} - /// Merge a peer's ILM expiry document into the local lifecycle config. /// /// Mirrors MinIO's `mergeWithCurrentLCConfig` with one hardening: incoming @@ -8029,15 +4002,6 @@ fn merge_incoming_lifecycle_config( }) } -/// True when the rule carries the expiry semantics that `replicateILMExpiry` -/// propagates. Del-marker expiration and abort-multipart are deliberately -/// excluded: MinIO's sender never emits them (`CloneNonTransition` drops -/// both), so treating them as traveling state would let a MinIO peer's -/// broadcast delete this site's del-marker-only rules. -fn lifecycle_rule_has_expiry(rule: &s3s::dto::LifecycleRule) -> bool { - rule.expiration.is_some() || rule.noncurrent_version_expiration.is_some() -} - fn lifecycle_rule_has_transition(rule: &s3s::dto::LifecycleRule) -> bool { rule.transitions.as_ref().is_some_and(|transitions| !transitions.is_empty()) || rule @@ -8046,73 +4010,6 @@ fn lifecycle_rule_has_transition(rule: &s3s::dto::LifecycleRule) -> bool { .is_some_and(|transitions| !transitions.is_empty()) } -/// Remove the fields that never travel between sites (MinIO -/// `CloneNonTransition` parity). -fn strip_site_local_lifecycle_fields(rule: &mut s3s::dto::LifecycleRule) { - rule.transitions = None; - rule.noncurrent_version_transitions = None; - rule.abort_incomplete_multipart_upload = None; - rule.del_marker_expiration = None; -} - -/// Reduce a lifecycle XML document to the expiry subset that is allowed to -/// travel between sites (what MinIO's sender emits): transition fields are -/// stripped and rules left with no expiry semantics are dropped. Returns -/// `None` when nothing remains — the receiver then merges with the empty set, -/// which is exactly the "no expiry rules here" statement. A document that -/// fails to parse is forwarded unfiltered (`Some(original)`): the receiver -/// merge strips it anyway, and turning a local parse error into a `None` -/// would delete the peers' replicated expiry rules. -fn lifecycle_expiry_subset_xml(raw: &[u8]) -> Option> { - if raw.is_empty() { - return None; - } - let config: s3s::dto::BucketLifecycleConfiguration = match deserialize(raw) { - Ok(config) => config, - Err(err) => { - warn!("failed to parse local lifecycle config for expiry replication; forwarding unfiltered: {err}"); - return Some(raw.to_vec()); - } - }; - let expiry_updated_at = config.expiry_updated_at.clone(); - let rules: Vec = config - .rules - .into_iter() - .filter_map(|mut rule| { - strip_site_local_lifecycle_fields(&mut rule); - lifecycle_rule_has_expiry(&rule).then_some(rule) - }) - .collect(); - if rules.is_empty() { - return None; - } - let subset = s3s::dto::BucketLifecycleConfiguration { - rules, - expiry_updated_at, - }; - match serialize(&subset) { - Ok(data) => Some(data), - Err(err) => { - warn!("failed to serialize lifecycle expiry subset; forwarding unfiltered: {err}"); - Some(raw.to_vec()) - } - } -} - -/// The expiry replication axis persisted in a lifecycle XML document, if any. -/// Used for the SRInfo bucket entry so bootstrap/repair items carry the -/// expiry axis instead of the whole-config write time (which local -/// transition-only edits inflate). -fn lifecycle_expiry_updated_at(raw: &[u8]) -> Option { - if raw.is_empty() { - return None; - } - deserialize::(raw) - .ok() - .and_then(|config| config.expiry_updated_at) - .map(OffsetDateTime::from) -} - /// The timestamp an incoming lc-config item must beat to be applied. /// /// - Present config with the expiry axis: the axis itself. @@ -8213,48 +4110,6 @@ fn is_zero_rule_lifecycle_tombstone(raw: &[u8]) -> bool { well_formed_document && quick_xml::de::from_reader::<_, Tombstone>(raw).is_ok() } -/// The ILM expiry statement this site contributes to its SRInfo bucket entry -/// (feeding bootstrap/repair and consistency views), if any. -/// `Some((subset_b64, axis))` — a `None` subset means "expiry rules were -/// removed at `axis`" and travels as an explicit timestamped delete item, so -/// a peer that missed the live delete still converges on repair. -fn lifecycle_expiry_statement( - metadata: &crate::admin::storage_api::bucket::metadata::BucketMetadata, -) -> Option<(Option, OffsetDateTime)> { - if metadata.lifecycle_config_xml.is_empty() { - // Deleted vs never configured: the whole-config write time survives - // deletion in bucket metadata and strictly exceeds the created-time - // backfill only after a real write. - return (metadata.lifecycle_config_updated_at > metadata.created).then_some((None, metadata.lifecycle_config_updated_at)); - } - let axis = lifecycle_expiry_updated_at(&metadata.lifecycle_config_xml); - match lifecycle_expiry_subset_xml(&metadata.lifecycle_config_xml) { - Some(subset) => { - // Legacy documents predate the axis field; their whole-config - // write time bounds the last expiry edit. - let axis = axis.unwrap_or(metadata.lifecycle_config_updated_at); - Some((raw_config_to_base64(&subset), axis)) - } - // Transition-only config: with an expiry axis the site once had - // expiry rules and properly removed them — the delete travels at - // that axis. Without one there is nothing to say (a delete stamped - // off the whole-config time would let a local transition edit erase - // newer peer expiry state). - None => axis.map(|axis| (None, axis)), - } -} - -/// Whether `rule` is in the shape the reconciler derives (`site-repl-` -/// naming the deployment its ARN targets). The reconciler rebuilds every such -/// rule from the current peer set — current peer or not, so a leftover from a -/// removed peer or a self-pointing rule is rebuilt away — while the merges -/// keep only the current peers' rules and treat a leftover as operator state -/// the edit replaces. An operator-authored `site-repl-*` id on an operator -/// ARN is outside the shape and survives every pass. -fn is_derived_site_replication_rule(rule: &ReplicationRule) -> bool { - site_replication_rule_deployment_id(rule).is_some() -} - fn replication_rule_deployment_id(rule: &ReplicationRule) -> Option { if let Some(rule_id) = rule.id.as_deref() { if let Some(deployment_id) = rule_id.strip_prefix("site-repl-") @@ -8304,123 +4159,6 @@ fn prune_removed_site_replication_rules( (Some(config), removed) } -fn build_site_replication_rule(arn: &str, priority: i32, rule_id: &str) -> ReplicationRule { - ReplicationRule { - delete_marker_replication: Some(DeleteMarkerReplication { - status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)), - }), - delete_replication: Some(DeleteReplication { - status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED), - }), - destination: Destination { - bucket: arn.to_string(), - ..Default::default() - }, - existing_object_replication: Some(ExistingObjectReplication { - status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED), - }), - filter: None, - id: Some(rule_id.to_string()), - prefix: None, - priority: Some(priority), - source_selection_criteria: Some(SourceSelectionCriteria { - replica_modifications: Some(ReplicaModifications { - status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED), - }), - sse_kms_encrypted_objects: None, - }), - status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED), - } -} - -fn build_site_replication_config( - bucket: &str, - state: &SiteReplicationState, - local_peer: &PeerInfo, - service_account_secret_key: &str, - existing: Option<&ReplicationConfiguration>, -) -> S3Result> { - // Reuse the ARN already recorded for a peer so the rule keeps pointing at the same - // bucket target `reconcile_site_replication_bucket_targets` keys off (a MinIO-era - // `arn:minio:...` target would otherwise be orphaned by a freshly minted ARN). - let configured_arns = site_replication_target_arns_by_peer(existing); - let mut rules = Vec::new(); - for peer in state.peers.values() { - if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { - continue; - } - - let Some(target) = site_replication_bucket_target_for_peer( - bucket, - state, - peer, - service_account_secret_key, - configured_arns.get(&peer.deployment_id).cloned(), - )? - else { - continue; - }; - rules.push(build_site_replication_rule( - &target.arn, - (rules.len() + 1) as i32, - &format!("site-repl-{}", peer.deployment_id), - )); - } - - if rules.is_empty() { - Ok(None) - } else { - Ok(Some(ReplicationConfiguration { - role: String::new(), - rules, - })) - } -} - -async fn ensure_site_replication_bucket_targets_with_runtime( - bucket: &str, - state: &SiteReplicationState, - local_peer: &PeerInfo, - config: Option<&s3s::dto::ReplicationConfiguration>, - service_account_secret_key: &str, - expected_incarnation_id: Uuid, -) -> S3Result<()> { - let existing = match metadata_sys::list_bucket_targets(bucket).await { - Ok(targets) => targets, - Err(StorageError::ConfigNotFound) => BucketTargets::default(), - Err(err) => return Err(ApiError::from(err).into()), - }; - let existing_json = serde_json::to_vec(&existing) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize bucket targets failed: {e}")))?; - - let updated = - reconcile_site_replication_bucket_targets(existing, bucket, state, local_peer, config, service_account_secret_key)?; - if updated.targets.is_empty() { - return Ok(()); - } - - let json_targets = serde_json::to_vec(&updated) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize bucket targets failed: {e}")))?; - // Rewriting identical targets would churn bucket metadata and rebuild every remote S3 - // client — noticeable now that startup reconciles all buckets, not just the one bucket - // an operation touched. - if json_targets == existing_json { - return Ok(()); - } - metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id) - .await - .map_err(ApiError::from)?; - Ok(()) -} - -async fn bucket_replication_config_for_target_refresh(bucket: &str) -> S3Result> { - match metadata_sys::get_replication_config(bucket).await { - Ok((config, _)) => Ok(Some(config)), - Err(StorageError::ConfigNotFound) => Ok(None), - Err(err) => Err(ApiError::from(err).into()), - } -} - async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> { let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) .await @@ -8441,65 +4179,6 @@ async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> { .await } -async fn ensure_site_replication_bucket_replication_config_with_runtime( - bucket: &str, - state: &SiteReplicationState, - local_peer: &PeerInfo, - service_account_secret_key: &str, - expected_incarnation_id: Uuid, -) -> S3Result<()> { - let existing = match metadata_sys::get_replication_config(bucket).await { - Ok((existing, _)) => Some(existing), - Err(StorageError::ConfigNotFound) => None, - Err(err) => return Err(ApiError::from(err).into()), - }; - - let Some(desired) = build_site_replication_config(bucket, state, local_peer, service_account_secret_key, existing.as_ref())? - else { - return Ok(()); - }; - - // Derived rules are state owned by this site: rebuild them from the current peer - // set on every pass instead of preserving whatever is on disk. A rule left over - // from a removed peer — or one whose destination ARN names this very deployment, - // which no bucket target can ever satisfy — must not survive, otherwise objects - // are queued against an ARN that resolves to nothing. - let (existing_role, existing_rules) = existing - .map(|config| (config.role, config.rules)) - .unwrap_or_else(|| (String::new(), Vec::new())); - let mut rules: Vec = existing_rules - .iter() - .filter(|rule| !is_derived_site_replication_rule(rule)) - .cloned() - .collect(); - rules.extend(desired.rules); - // Operator priorities are the operator's policy; only the derived rules - // take free slots, by the same function as the config merges so a merged - // write and this pass agree byte for byte. - assign_site_replication_rule_priorities(&mut rules, is_derived_site_replication_rule); - - // Only a `role` naming a current peer is ours to drop — an operator-authored role is - // part of the bucket's S3-visible configuration, and repairing a reverse rule must not - // quietly rewrite it. Same rule as `merge_incoming_replication_config`. - let role = if is_site_replication_role(&existing_role, &remote_peer_deployment_ids(state, local_peer)) { - String::new() - } else { - existing_role.clone() - }; - - if rules == existing_rules && role == existing_role { - return Ok(()); - } - - let data = serialize(&ReplicationConfiguration { role, rules }) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}")))?; - metadata_sys::update_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) - .await - .map_err(ApiError::from)?; - - Ok(()) -} - async fn ensure_site_replication_bucket_setup(bucket: &str) -> S3Result { let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(false); @@ -8519,40 +4198,6 @@ async fn ensure_site_replication_bucket_setup_for_incarnation(bucket: &str, inca Ok(true) } -async fn ensure_site_replication_bucket_setup_with_runtime(bucket: &str, runtime: &SiteReplicationRuntime) -> S3Result<()> { - let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) - .await - .map_err(ApiError::from)?; - ensure_site_replication_bucket_setup_with_runtime_for_incarnation(bucket, runtime, expected_incarnation_id).await -} - -async fn ensure_site_replication_bucket_setup_with_runtime_for_incarnation( - bucket: &str, - runtime: &SiteReplicationRuntime, - expected_incarnation_id: Uuid, -) -> S3Result<()> { - let _targets_guard = lock_bucket_targets_metadata(bucket).await; - let config = bucket_replication_config_for_target_refresh(bucket).await?; - ensure_site_replication_bucket_targets_with_runtime( - bucket, - &runtime.state, - &runtime.local_peer, - config.as_ref(), - &runtime.service_account_secret_key, - expected_incarnation_id, - ) - .await?; - ensure_site_replication_bucket_replication_config_with_runtime( - bucket, - &runtime.state, - &runtime.local_peer, - &runtime.service_account_secret_key, - expected_incarnation_id, - ) - .await?; - Ok(()) -} - async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deployment_ids: &HashSet) -> S3Result { let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) .await @@ -9247,31 +4892,6 @@ fn apply_state_edit_req(mut state: SiteReplicationState, body: SRStateEditReq) - state } -fn bucket_versioning_xml() -> S3Result> { - let config = VersioningConfiguration { - status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), - ..Default::default() - }; - serialize(&config).map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize versioning failed: {e}"))) -} - -async fn ensure_site_replication_bucket_versioning(bucket: &str) -> S3Result<()> { - let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) - .await - .map_err(ApiError::from)?; - match metadata_sys::get_versioning_config(bucket).await { - Ok((config, _)) if config.enabled() => return Ok(()), - Ok(_) | Err(StorageError::ConfigNotFound) => {} - Err(err) => return Err(ApiError::from(err).into()), - } - - metadata_sys::update_if_incarnation(bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?, expected_incarnation_id) - .await - .map_err(ApiError::from)?; - - Ok(()) -} - fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option) -> bool { incoming_updated_at.is_some_and(|incoming_updated_at| incoming_updated_at < local_updated_at) } @@ -11734,12 +7354,19 @@ impl Operation for SRRotateServiceAccountHandler { #[cfg(test)] mod tests { use super::*; - use crate::admin::runtime_sources::{current_outbound_tls_generation, set_test_outbound_tls_generation}; use crate::admin::storage_api::runtime::Endpoint; use crate::admin::storage_api::runtime::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::site_replication::identity::deployment_id_for_endpoint; + use crate::storage_api::site_replication::is_site_replication_role; use axum::{Router, extract::State, routing::any}; + use base64_simd::STANDARD as BASE64_STANDARD; use http::{HeaderMap, HeaderValue, Uri}; + use rustfs_config::ENV_RUSTFS_TLS_PATH; + use rustfs_iam::store::{UserType, sr_wire_user_type}; + use rustfs_madmin::{AddOrUpdateUserReq, SRBucketInfo, SRIAMPolicy, SRRetryStats}; use rustfs_policy::policy::action::S3Action; + use rustfs_tls_runtime::GlobalPublishedOutboundTlsState; + use s3s::dto::VersioningConfiguration; use serial_test::serial; use std::sync::{ Arc, Mutex as StdMutex, @@ -11748,6 +7375,7 @@ mod tests { use temp_env::with_var; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; + use url::Url; #[test] fn sts_replication_restores_groups_from_signed_claims() { @@ -17902,42 +13530,6 @@ mod tests { assert!(err.to_string().contains("cached error"), "expected cached error detail, got: {}", err); } - #[tokio::test] - #[serial] - async fn test_site_replication_peer_client_rebuilds_when_generation_changes() { - let previous_generation = current_outbound_tls_generation().0; - let previous_cache = { - let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; - let snapshot = cache.clone(); - *cache = None; - snapshot - }; - - set_test_outbound_tls_generation(101); - site_replication_peer_client() - .await - .expect("initial client build should succeed"); - let cache = SITE_REPLICATION_PEER_CLIENT.lock().await; - let cached = cache.as_ref().expect("cache should be populated"); - assert_eq!(cached.generation, 101); - assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_))); - drop(cache); - - set_test_outbound_tls_generation(102); - site_replication_peer_client() - .await - .expect("new generation should rebuild client"); - let cache = SITE_REPLICATION_PEER_CLIENT.lock().await; - let cached = cache.as_ref().expect("cache should be populated"); - assert_eq!(cached.generation, 102); - assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_))); - - drop(cache); - set_test_outbound_tls_generation(previous_generation); - let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; - *cache = previous_cache; - } - #[test] fn test_site_repl_netperf_reports_unsupported_without_measurements() { let result = unsupported_site_netperf_result("https://peer.example.com".to_string()); diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 701c87839..f685f9d4a 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -23,8 +23,6 @@ pub(crate) mod route_policy; pub mod router; pub(crate) mod runtime_sources; pub mod service; -pub mod site_replication_identity; -pub(crate) mod site_replication_state; pub(crate) mod storage_api; pub mod utils; diff --git a/rustfs/src/admin/runtime_sources.rs b/rustfs/src/admin/runtime_sources.rs index 0d053bd52..938b0ea86 100644 --- a/rustfs/src/admin/runtime_sources.rs +++ b/rustfs/src/admin/runtime_sources.rs @@ -36,10 +36,8 @@ pub(crate) use crate::runtime_sources::{ }; use rustfs_config::server_config::Config; use rustfs_kms::KmsServiceManager; -use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration}; +use rustfs_tls_runtime::GlobalPublishedOutboundTlsState; use std::sync::Arc; -#[cfg(test)] -use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::RwLock; pub(crate) fn default_admin_usecase() -> DefaultAdminUsecase { @@ -116,29 +114,6 @@ pub(crate) fn current_or_init_kms_runtime_service_manager() -> Arc TlsGeneration { - root_runtime_sources::current_outbound_tls_generation().unwrap_or_else(empty_outbound_tls_generation) -} - -#[cfg(test)] -fn empty_outbound_tls_generation() -> TlsGeneration { - TlsGeneration(TEST_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed)) -} - -#[cfg(not(test))] -fn empty_outbound_tls_generation() -> TlsGeneration { - TlsGeneration(0) -} - pub(crate) async fn current_outbound_tls_state() -> GlobalPublishedOutboundTlsState { if let Some(state) = root_runtime_sources::current_outbound_tls_state().await { return state; diff --git a/rustfs/src/admin/service/site_replication.rs b/rustfs/src/admin/service/site_replication.rs index db6fdde22..3f47b584e 100644 --- a/rustfs/src/admin/service/site_replication.rs +++ b/rustfs/src/admin/service/site_replication.rs @@ -13,11 +13,11 @@ // limitations under the License. use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context}; -use crate::admin::site_replication_identity::{ +use crate::admin::storage_api::error::Error as StorageError; +use crate::site_replication::identity::{ deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with, }; -use crate::admin::site_replication_state::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock_on}; -use crate::admin::storage_api::error::Error as StorageError; +use crate::site_replication::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock_on}; use crate::storage::storage_api::{read_config_no_lock, save_config_no_lock}; use rustfs_madmin::PeerInfo; use s3s::{S3Error, S3ErrorCode, S3Result}; diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 3e2421f93..1c8a14c26 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -445,8 +445,8 @@ pub(crate) mod replication { pub(crate) use super::ecstore_bucket::replication::{ OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, - REPLICATION_WRITABLE_FIELDS, assign_site_replication_rule_priorities, is_site_replication_role, - merge_incoming_replication_config, replication_target_arn_deployment_id, site_replication_rule_deployment_id, + REPLICATION_WRITABLE_FIELDS, assign_site_replication_rule_priorities, merge_incoming_replication_config, + replication_target_arn_deployment_id, }; pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus; pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats; @@ -653,8 +653,6 @@ pub(crate) mod replication { pub(crate) mod target { pub(crate) use super::ecstore_bucket::target::duration_from_secs_or_nanos; - #[allow(clippy::upper_case_acronyms)] - pub(crate) type ARN = super::ecstore_bucket::target::ARN; pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget; pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType; pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 0bc2a3c0a..b7d526fd1 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -64,9 +64,6 @@ use super::storage_api::bucket_usecase::{ get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, request_context, validate_list_object_unordered_with_delimiter, }; -use crate::admin::handlers::site_replication::{ - site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook, -}; use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete; use crate::app::runtime_sources::{ AppContext, current_app_context, current_encryption_service, current_notification_system, @@ -75,6 +72,9 @@ use crate::app::runtime_sources::{ use crate::auth::get_condition_values_with_client_info; use crate::error::ApiError; use crate::shared_types::RemoteAddr; +use crate::site_replication::{ + site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook, +}; use crate::storage::storage_api::lock_bucket_targets_metadata; use http::StatusCode; use metrics::counter; diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index 1b6fac1fa..95b0c0fba 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -100,6 +100,7 @@ pub mod runtime_capabilities; pub(crate) mod runtime_sources; pub mod server; pub mod shared_types; +pub(crate) mod site_replication; pub(crate) mod site_replication_reconcile; pub(crate) mod startup_audit; pub(crate) mod startup_auth; diff --git a/rustfs/src/site_replication/hooks.rs b/rustfs/src/site_replication/hooks.rs new file mode 100644 index 000000000..ee03083a6 --- /dev/null +++ b/rustfs/src/site_replication/hooks.rs @@ -0,0 +1,1158 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub(crate) const SITE_REPLICATION_PEER_BUCKET_OPS_PATH: &str = "/rustfs/admin/v3/site-replication/peer/bucket-ops"; + +pub(crate) const SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING: &str = "make-with-versioning"; + +pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "configure-replication"; + +pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock> = LazyLock::new(|| RwLock::new(())); + +#[derive(Debug, Default)] +pub(crate) struct SiteReplicationBootstrapPlan { + pub(crate) iam_items: Vec, + pub(crate) bucket_make_ops: Vec, + pub(crate) bucket_items: Vec, + pub(crate) bucket_configure_ops: Vec, +} + +pub(crate) fn bootstrap_bucket_op_path(bucket: &str, operation: &str) -> String { + format!( + "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", + form_urlencoded::Serializer::new(String::new()) + .append_pair("bucket", bucket) + .append_pair("operation", operation) + .finish() + ) +} + +pub(crate) fn with_site_replication_bootstrap_token(path: &str, token: &str) -> String { + let separator = if path.contains('?') { '&' } else { '?' }; + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("bootstrapToken", token) + .finish(); + format!("{path}{separator}{query}") +} + +/// Query for a peer `make-with-versioning` bucket op. `versioningEnabled` +/// always travels so the outbound query matches MinIO's site-replication +/// make-bucket wire contract: MinIO's own create-bucket hook sends +/// `versioningEnabled=true` on this op. RustFS's inbound handler +/// force-enables versioning either way. +pub(crate) fn make_with_versioning_bucket_op_path(bucket: &str, created_at: Option<&str>, lock_enabled: bool) -> String { + let mut query = form_urlencoded::Serializer::new(String::new()); + query.append_pair("bucket", bucket); + query.append_pair("operation", SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING); + query.append_pair("versioningEnabled", "true"); + if let Some(created_at) = created_at { + query.append_pair("createdAt", created_at); + } + if lock_enabled { + query.append_pair("lockEnabled", "true"); + } + format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?{}", query.finish()) +} + +pub(crate) fn bootstrap_bucket_make_op_path(bucket: &SRBucketInfo) -> String { + let created_at = bucket + .created_at + .and_then(|value| value.format(&time::format_description::well_known::Rfc3339).ok()); + make_with_versioning_bucket_op_path(&bucket.bucket, created_at.as_deref(), bucket.object_lock_config.is_some()) +} + +pub(crate) fn bootstrap_bucket_meta_item( + bucket: &SRBucketInfo, + item_type: &str, + updated_at: Option, +) -> SRBucketMeta { + SRBucketMeta { + bucket: bucket.bucket.clone(), + r#type: item_type.to_string(), + updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + derived_rule_contract: true, + ..Default::default() + } +} + +pub(crate) fn bootstrap_bucket_quota_value(bucket: &str, raw: &str) -> S3Result { + serde_json::from_slice(&decode_bucket_meta_wire_value(raw)) + .map_err(|e| s3_error!(InvalidRequest, "invalid quota metadata for bootstrap bucket `{bucket}`: {e}")) +} + +pub(crate) fn append_bootstrap_bucket_item( + items: &mut Vec, + bucket: &SRBucketInfo, + item_type: &str, + value: Option, + updated_at: Option, + apply: impl FnOnce(&mut SRBucketMeta, String) -> S3Result<()>, +) -> S3Result<()> { + if let Some(value) = value { + let mut item = bootstrap_bucket_meta_item(bucket, item_type, updated_at); + apply(&mut item, value)?; + items.push(item); + } + Ok(()) +} + +pub(crate) fn append_bootstrap_bucket_items( + plan: &mut SiteReplicationBootstrapPlan, + bucket: &SRBucketInfo, + replicate_ilm_expiry: bool, +) -> S3Result<()> { + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "policy", + bucket.policy.clone().map(|value| value.to_string()), + bucket.policy_updated_at, + |item, value| { + item.policy = + Some(serde_json::from_str(&value).map_err(|e| { + s3_error!(InvalidRequest, "invalid bucket policy for bootstrap bucket `{}`: {e}", item.bucket) + })?); + Ok(()) + }, + )?; + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "version-config", + bucket.versioning.clone(), + bucket.versioning_config_updated_at, + |item, value| { + item.versioning = Some(value); + Ok(()) + }, + )?; + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "tags", + bucket.tags.clone(), + bucket.tag_config_updated_at, + |item, value| { + item.tags = Some(value); + Ok(()) + }, + )?; + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "object-lock-config", + bucket.object_lock_config.clone(), + bucket.object_lock_config_updated_at, + |item, value| { + item.object_lock_config = Some(value); + Ok(()) + }, + )?; + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "sse-config", + bucket.sse_config.clone(), + bucket.sse_config_updated_at, + |item, value| { + item.sse_config = Some(value); + Ok(()) + }, + )?; + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "replication-config", + bucket.replication_config.clone(), + bucket.replication_config_updated_at, + |item, value| { + item.replication_config = Some(value); + Ok(()) + }, + )?; + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "quota-config", + bucket.quota_config.clone(), + bucket.quota_config_updated_at, + |item, value| { + item.quota = Some(bootstrap_bucket_quota_value(&item.bucket, &value)?); + Ok(()) + }, + )?; + if replicate_ilm_expiry { + if bucket.expiry_lc_config.is_some() { + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "lc-config", + bucket.expiry_lc_config.clone(), + bucket.expiry_lc_config_updated_at, + |item, value| { + item.expiry_lc_config = Some(value); + // `updated_at` here is the entry's expiry axis (see the + // SRBucketInfo construction), not the wall clock. + item.expiry_updated_at = item.updated_at; + Ok(()) + }, + )?; + } else if bucket.expiry_lc_config_updated_at.is_some() { + // Expiry rules were removed at this axis (lifecycle_expiry_statement): + // an explicit timestamped delete item, so a peer that missed the + // live delete converges on bootstrap/repair instead of keeping + // stale expiry rules. The receiver's staleness guard protects a + // peer whose expiry state is newer. + let mut item = bootstrap_bucket_meta_item(bucket, "lc-config", bucket.expiry_lc_config_updated_at); + item.expiry_updated_at = item.updated_at; + plan.bucket_items.push(item); + } + } + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "cors-config", + bucket.cors_config.clone(), + bucket.cors_config_updated_at, + |item, value| { + item.cors = Some(value); + Ok(()) + }, + ) +} + +pub(crate) fn group_status_from_desc(status: &str) -> GroupStatus { + if status.eq_ignore_ascii_case("disabled") { + GroupStatus::Disabled + } else { + GroupStatus::Enabled + } +} + +pub(crate) fn site_replication_info_replicates_ilm_expiry(info: &SRInfo) -> bool { + info.state.peers.values().any(|peer| peer.replicate_ilm_expiry) +} + +pub(crate) fn site_replication_state_replicates_ilm_expiry(state: &SiteReplicationState) -> bool { + state.peers.values().any(|peer| peer.replicate_ilm_expiry) +} + +pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result { + let mut plan = SiteReplicationBootstrapPlan::default(); + let replicate_ilm_expiry = site_replication_info_replicates_ilm_expiry(info); + + for (name, policy) in &info.policies { + plan.iam_items.push(SRIAMItem { + r#type: "policy".to_string(), + name: name.clone(), + policy: policy.policy.clone(), + updated_at: policy.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }); + } + + for (access_key, user) in &info.user_info_map { + if let Some(secret_key) = &user.secret_key { + plan.iam_items.push(SRIAMItem { + r#type: "iam-user".to_string(), + iam_user: Some(rustfs_madmin::SRIAMUser { + access_key: access_key.clone(), + is_delete_req: false, + user_req: Some(AddOrUpdateUserReq { + secret_key: secret_key.clone(), + policy: user.policy_name.clone(), + status: user.status.clone(), + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: user.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }); + } + } + + for (name, desc) in &info.group_desc_map { + plan.iam_items.push(SRIAMItem { + r#type: "group-info".to_string(), + group_info: Some(SRGroupInfo { + update_req: GroupAddRemove { + group: if desc.name.is_empty() { + name.clone() + } else { + desc.name.clone() + }, + members: desc.members.clone(), + status: group_status_from_desc(&desc.status), + is_remove: false, + }, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: desc.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }); + } + + for mapping in info.user_policies.values().chain(info.group_policies.values()) { + plan.iam_items.push(SRIAMItem { + r#type: "policy-mapping".to_string(), + policy_mapping: Some(mapping.clone()), + updated_at: mapping.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }); + } + + for bucket in info.buckets.values() { + plan.bucket_make_ops.push(bootstrap_bucket_make_op_path(bucket)); + append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?; + plan.bucket_configure_ops + .push(bootstrap_bucket_op_path(&bucket.bucket, "configure-replication")); + } + + Ok(plan) +} + +pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { + let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; + let runtime = { + // The bucket-op lock is what orders this against add/remove. The + // state is only read here (through the runtime snapshot), and the + // bucket setup below writes bucket metadata, never the state object — + // holding the state transaction across it would put local metadata + // IO inside a distributed lock for nothing. + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + + ensure_site_replication_bucket_versioning(bucket).await?; + ensure_site_replication_bucket_setup_with_runtime(bucket, &runtime).await?; + runtime + }; + + broadcast_site_replication_make_bucket(bucket, lock_enabled, Some(&runtime), None).await +} + +pub(crate) async fn broadcast_site_replication_json_using_runtime( + runtime: Option<&SiteReplicationRuntime>, + path: &str, + body: &T, +) -> S3Result<()> { + match runtime { + Some(runtime) => broadcast_site_replication_json_with_runtime(runtime, path, body).await, + None => broadcast_site_replication_json(path, body).await, + } +} + +pub(crate) async fn broadcast_site_replication_make_bucket( + bucket: &str, + lock_enabled: bool, + runtime: Option<&SiteReplicationRuntime>, + bootstrap_token: Option<&str>, +) -> S3Result<()> { + let created_at = current_object_store_handle() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))? + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)? + .created + .unwrap_or_else(OffsetDateTime::now_utc) + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_default(); + + let path = make_with_versioning_bucket_op_path(bucket, Some(&created_at), lock_enabled); + let path = if let Some(token) = bootstrap_token { + with_site_replication_bootstrap_token(&path, token) + } else { + path + }; + broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await?; + + let configure_path = bootstrap_bucket_op_path(bucket, "configure-replication"); + let configure_path = if let Some(token) = bootstrap_token { + with_site_replication_bootstrap_token(&configure_path, token) + } else { + configure_path + }; + broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await +} + +pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> { + let operation = if force_delete { + "force-delete-bucket" + } else { + "delete-bucket" + }; + let path = format!( + "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", + form_urlencoded::Serializer::new(String::new()) + .append_pair("bucket", bucket) + .append_pair("operation", operation) + .finish() + ); + broadcast_site_replication_json(&path, &serde_json::json!({})).await +} + +pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if item.r#type == "lc-config" && !site_replication_state_replicates_ilm_expiry(&runtime.state) { + return Ok(()); + } + if item.r#type == "lc-config" { + // Only the expiry subset travels (MinIO peers install incoming rules + // verbatim, so transition rules must never leave this site). An empty + // subset becomes a delete, which the receiver merges with the empty + // set — local transition rules there survive. + item.expiry_lc_config = item + .expiry_lc_config + .and_then(|raw| lifecycle_expiry_subset_xml(raw.as_bytes())) + .map(|data| String::from_utf8_lossy(&data).into_owned()); + } + broadcast_site_replication_json_with_runtime( + &runtime, + "/rustfs/admin/v3/site-replication/peer/bucket-meta", + &encode_bucket_meta_wire_item(item), + ) + .await +} + +pub async fn site_replication_iam_change_hook(item: SRIAMItem) -> S3Result<()> { + broadcast_site_replication_json("/rustfs/admin/v3/site-replication/peer/iam-item", &item).await +} + +pub(crate) fn raw_config_to_string(raw: &[u8]) -> Option { + if raw.is_empty() { + return None; + } + String::from_utf8(raw.to_vec()).ok() +} + +pub(crate) fn raw_config_to_base64(raw: &[u8]) -> Option { + (!raw.is_empty()).then(|| BASE64_STANDARD.encode_to_string(raw)) +} + +pub(crate) fn encode_bucket_meta_wire_value(value: Option) -> Option { + value.map(|raw| BASE64_STANDARD.encode_to_string(raw.as_bytes())) +} + +pub(crate) fn encode_bucket_meta_wire_item(mut item: SRBucketMeta) -> SRBucketMeta { + item.versioning = encode_bucket_meta_wire_value(item.versioning); + item.tags = encode_bucket_meta_wire_value(item.tags); + item.object_lock_config = encode_bucket_meta_wire_value(item.object_lock_config); + item.sse_config = encode_bucket_meta_wire_value(item.sse_config); + item.replication_config = encode_bucket_meta_wire_value(item.replication_config); + item.expiry_lc_config = encode_bucket_meta_wire_value(item.expiry_lc_config); + item.cors = encode_bucket_meta_wire_value(item.cors); + item +} + +pub(crate) fn decode_bucket_meta_wire_value(raw: &str) -> Vec { + BASE64_STANDARD + .decode_to_vec(raw.as_bytes()) + .ok() + .filter(|decoded| std::str::from_utf8(decoded).is_ok()) + .unwrap_or_else(|| raw.as_bytes().to_vec()) +} + +pub(crate) fn decode_bucket_meta_wire_option(value: Option) -> Option> { + value.map(|raw| decode_bucket_meta_wire_value(&raw)) +} + +pub(crate) fn maybe_time(value: OffsetDateTime) -> Option { + (value != OffsetDateTime::UNIX_EPOCH).then_some(value) +} + +pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result { + let Some(store) = current_object_store_handle() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let mut info = SRInfo { + enabled: state.enabled(), + name: local_peer.name.clone(), + deployment_id: local_peer.deployment_id.clone(), + state: SRStateInfo { + name: local_peer.name.clone(), + peers: state.peers.clone(), + updated_at: state.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + + let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?; + for bucket in buckets { + let metadata = metadata_sys::get(&bucket.name).await.ok(); + let mut entry = SRBucketInfo { + bucket: bucket.name.clone(), + created_at: bucket.created, + location: current_region().map(|region| region.to_string()).unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + + if let Some(metadata) = metadata { + entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); + entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml); + entry.tags = raw_config_to_base64(&metadata.tagging_config_xml); + entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml); + entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); + entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); + entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); + // Expiry subset only: this entry feeds both the bootstrap/repair + // plan (peers must not receive transition rules) and cross-site + // consistency views (transition rules are site-local and would + // read as false mismatches). A deleted expiry state is a `None` + // value with the deletion's axis so repair can converge peers + // that missed the live delete. + let expiry_statement = lifecycle_expiry_statement(&metadata); + entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); + entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); + entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); + entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); + entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); + entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); + entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); + entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); + entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); + // The expiry axis, not the whole-config write time: local + // transition-only edits inflate the latter, and a repair item + // stamped with it could out-rank a newer real expiry edit on a + // third site. + entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); + entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); + entry.replication_targets_online = + Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await); + } + + info.buckets.insert(bucket.name, entry); + } + + if let Some(iam_sys) = current_iam_handle() { + for (name, policy_doc) in iam_sys.list_policy_docs("").await.map_err(ApiError::from)? { + info.policies.insert( + name, + SRIAMPolicy { + policy: serde_json::to_value(policy_doc.policy).ok(), + updated_at: policy_doc.update_date, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + ); + } + + let users = iam_sys.list_users().await.map_err(ApiError::from)?; + for (name, user) in users { + info.user_info_map.insert(name, user); + } + + let groups = iam_sys.list_groups_load().await.map_err(ApiError::from)?; + for group in groups { + let desc = iam_sys.get_group_description(&group).await.map_err(ApiError::from)?; + info.group_desc_map.insert(group.clone(), desc); + } + + let mut user_policies = HashMap::::new(); + iam_sys + .load_mapped_policies(UserType::Reg, false, &mut user_policies) + .await + .map_err(ApiError::from)?; + for (name, mapping) in user_policies { + info.user_policies + .insert(name.clone(), mapped_policy_to_sr_mapping(name, false, UserType::Reg, mapping)); + } + + let mut group_policies = HashMap::::new(); + iam_sys + .load_mapped_policies(UserType::None, true, &mut group_policies) + .await + .map_err(ApiError::from)?; + for (name, mapping) in group_policies { + info.group_policies + .insert(name.clone(), mapped_policy_to_sr_mapping(name, true, UserType::None, mapping)); + } + } + + for (name, bucket_info) in &info.buckets { + if let Some(raw) = bucket_info + .replication_config + .as_ref() + .and_then(|value| serde_json::from_str::(value).ok()) + { + info.replication_cfg.insert(name.clone(), raw); + } + } + + Ok(info) +} + +pub(crate) fn mapped_policy_to_sr_mapping( + name: String, + is_group: bool, + user_type: UserType, + mapping: MappedPolicy, +) -> SRPolicyMapping { + SRPolicyMapping { + user_or_group: name, + user_type: sr_wire_user_type(user_type, is_group), + is_group, + policy: mapping.policies, + updated_at: Some(mapping.update_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } +} + +pub(crate) fn bucket_target_endpoint(target: &BucketTarget) -> String { + let scheme = if target.secure { "https" } else { "http" }; + canonical_endpoint(&format!("{scheme}://{}", target.endpoint)) +} + +pub(crate) fn bucket_target_matches_peer(target: &BucketTarget, peer: &PeerInfo) -> bool { + if !target.deployment_id.is_empty() { + return target.deployment_id == peer.deployment_id; + } + bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint) +} + +pub(crate) fn site_replication_target_arns_by_peer(config: Option<&ReplicationConfiguration>) -> HashMap { + let mut arns_by_peer = HashMap::new(); + let Some(config) = config else { + return arns_by_peer; + }; + + let mut configured_arns = Vec::new(); + if !config.role.trim().is_empty() { + configured_arns.push(config.role.clone()); + } + for rule in &config.rules { + let arn = rule.destination.bucket.trim(); + if !arn.is_empty() { + configured_arns.push(arn.to_string()); + } + } + + for arn in configured_arns { + if let Some(deployment_id) = replication_target_arn_deployment_id(&arn) { + arns_by_peer.entry(deployment_id).or_insert(arn); + } + } + + arns_by_peer +} + +pub(crate) fn site_replication_bucket_target_for_peer( + bucket: &str, + state: &SiteReplicationState, + peer: &PeerInfo, + service_account_secret_key: &str, + arn_override: Option, +) -> S3Result> { + if state.service_account_access_key.is_empty() || service_account_secret_key.is_empty() { + return Ok(None); + } + + let parsed = Url::parse(&peer.endpoint) + .ok() + .or_else(|| Url::parse(&format!("http://{}", peer.endpoint.trim())).ok()) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {}", peer.endpoint)))?; + let host = parsed.host_str().ok_or_else(|| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("peer endpoint missing host: {}", peer.endpoint)) + })?; + let port = parsed.port_or_known_default().ok_or_else(|| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("peer endpoint missing port: {}", peer.endpoint)) + })?; + let region = current_region() + .map(|region| region.to_string()) + .filter(|region| !region.is_empty()) + .unwrap_or_else(|| "us-east-1".to_string()); + let arn = arn_override.unwrap_or_else(|| { + ARN::new( + BucketTargetType::ReplicationService, + peer.deployment_id.clone(), + String::new(), + bucket.to_string(), + ) + .to_string() + }); + + Ok(Some(BucketTarget { + source_bucket: bucket.to_string(), + endpoint: format!("{host}:{port}"), + credentials: Some(Credentials { + access_key: state.service_account_access_key.clone(), + secret_key: service_account_secret_key.to_string(), + session_token: None, + expiration: None, + }), + target_bucket: bucket.to_string(), + secure: parsed.scheme().eq_ignore_ascii_case("https"), + arn, + region, + target_type: BucketTargetType::ReplicationService, + deployment_id: peer.deployment_id.clone(), + skip_tls_verify: peer.skip_tls_verify, + ca_cert_pem: peer.ca_cert_pem.clone(), + ..Default::default() + })) +} + +pub(crate) fn reconcile_site_replication_bucket_targets( + existing: BucketTargets, + bucket: &str, + state: &SiteReplicationState, + local_peer: &PeerInfo, + config: Option<&ReplicationConfiguration>, + service_account_secret_key: &str, +) -> S3Result { + if !state.enabled() || state.service_account_access_key.is_empty() || service_account_secret_key.is_empty() { + return Ok(existing); + } + + let configured_arns = site_replication_target_arns_by_peer(config); + let mut targets = existing.targets; + + for peer in state.peers.values() { + if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { + continue; + } + + let Some(mut target) = site_replication_bucket_target_for_peer( + bucket, + state, + peer, + service_account_secret_key, + configured_arns.get(&peer.deployment_id).cloned(), + )? + else { + continue; + }; + + if let Some(index) = targets.iter().position(|existing| { + existing.target_type == BucketTargetType::ReplicationService + && (bucket_target_matches_peer(existing, peer) || existing.arn == target.arn) + }) { + let existing = targets[index].clone(); + target.path = existing.path; + target.region = existing.region; + target.bandwidth_limit = existing.bandwidth_limit; + target.replication_sync = existing.replication_sync; + target.storage_class = existing.storage_class; + target.health_check_duration = existing.health_check_duration; + target.disable_proxy = existing.disable_proxy; + target.reset_before_date = existing.reset_before_date; + target.reset_id = existing.reset_id; + target.total_downtime = existing.total_downtime; + target.last_online = existing.last_online; + target.online = existing.online; + target.latency = existing.latency; + target.edge = existing.edge; + target.edge_sync_before_expiry = existing.edge_sync_before_expiry; + target.offline_count = existing.offline_count; + targets[index] = target; + } else { + targets.push(target); + } + } + + Ok(BucketTargets { targets }) +} + +/// Whether every `site-repl-*` rule on this bucket resolves to a live remote target. +/// +/// The rule set alone cannot answer this: a rule can be perfectly formed while the endpoint +/// recorded for its peer is one this site cannot reach, so `update_all_targets` never built +/// a client for it and `replicate_object` drops every object against that ARN. Reads the +/// already-resolved client map rather than rebuilding clients, so it stays cheap enough for +/// the status path. +pub(crate) async fn site_replication_targets_online(bucket: &str, replication_config_xml: &[u8]) -> bool { + let Ok(config) = deserialize::(replication_config_xml) else { + return true; + }; + + for rule in config.rules.iter().filter(|rule| is_derived_site_replication_rule(rule)) { + if BucketTargetSys::get() + .get_remote_target_client_by_arn(bucket, &rule.destination.bucket) + .await + .is_none() + { + return false; + } + } + + true +} + +/// True when the rule carries the expiry semantics that `replicateILMExpiry` +/// propagates. Del-marker expiration and abort-multipart are deliberately +/// excluded: MinIO's sender never emits them (`CloneNonTransition` drops +/// both), so treating them as traveling state would let a MinIO peer's +/// broadcast delete this site's del-marker-only rules. +pub(crate) fn lifecycle_rule_has_expiry(rule: &LifecycleRule) -> bool { + rule.expiration.is_some() || rule.noncurrent_version_expiration.is_some() +} + +/// Remove the fields that never travel between sites (MinIO +/// `CloneNonTransition` parity). +pub(crate) fn strip_site_local_lifecycle_fields(rule: &mut LifecycleRule) { + rule.transitions = None; + rule.noncurrent_version_transitions = None; + rule.abort_incomplete_multipart_upload = None; + rule.del_marker_expiration = None; +} + +/// Reduce a lifecycle XML document to the expiry subset that is allowed to +/// travel between sites (what MinIO's sender emits): transition fields are +/// stripped and rules left with no expiry semantics are dropped. Returns +/// `None` when nothing remains — the receiver then merges with the empty set, +/// which is exactly the "no expiry rules here" statement. A document that +/// fails to parse is forwarded unfiltered (`Some(original)`): the receiver +/// merge strips it anyway, and turning a local parse error into a `None` +/// would delete the peers' replicated expiry rules. +pub(crate) fn lifecycle_expiry_subset_xml(raw: &[u8]) -> Option> { + if raw.is_empty() { + return None; + } + let config: BucketLifecycleConfiguration = match deserialize(raw) { + Ok(config) => config, + Err(err) => { + warn!("failed to parse local lifecycle config for expiry replication; forwarding unfiltered: {err}"); + return Some(raw.to_vec()); + } + }; + let expiry_updated_at = config.expiry_updated_at.clone(); + let rules: Vec = config + .rules + .into_iter() + .filter_map(|mut rule| { + strip_site_local_lifecycle_fields(&mut rule); + lifecycle_rule_has_expiry(&rule).then_some(rule) + }) + .collect(); + if rules.is_empty() { + return None; + } + let subset = BucketLifecycleConfiguration { + rules, + expiry_updated_at, + }; + match serialize(&subset) { + Ok(data) => Some(data), + Err(err) => { + warn!("failed to serialize lifecycle expiry subset; forwarding unfiltered: {err}"); + Some(raw.to_vec()) + } + } +} + +/// The expiry replication axis persisted in a lifecycle XML document, if any. +/// Used for the SRInfo bucket entry so bootstrap/repair items carry the +/// expiry axis instead of the whole-config write time (which local +/// transition-only edits inflate). +pub(crate) fn lifecycle_expiry_updated_at(raw: &[u8]) -> Option { + if raw.is_empty() { + return None; + } + deserialize::(raw) + .ok() + .and_then(|config| config.expiry_updated_at) + .map(OffsetDateTime::from) +} + +/// The ILM expiry statement this site contributes to its SRInfo bucket entry +/// (feeding bootstrap/repair and consistency views), if any. +/// `Some((subset_b64, axis))` — a `None` subset means "expiry rules were +/// removed at `axis`" and travels as an explicit timestamped delete item, so +/// a peer that missed the live delete still converges on repair. +pub(crate) fn lifecycle_expiry_statement( + metadata: &crate::storage_api::site_replication::BucketMetadata, +) -> Option<(Option, OffsetDateTime)> { + if metadata.lifecycle_config_xml.is_empty() { + // Deleted vs never configured: the whole-config write time survives + // deletion in bucket metadata and strictly exceeds the created-time + // backfill only after a real write. + return (metadata.lifecycle_config_updated_at > metadata.created).then_some((None, metadata.lifecycle_config_updated_at)); + } + let axis = lifecycle_expiry_updated_at(&metadata.lifecycle_config_xml); + match lifecycle_expiry_subset_xml(&metadata.lifecycle_config_xml) { + Some(subset) => { + // Legacy documents predate the axis field; their whole-config + // write time bounds the last expiry edit. + let axis = axis.unwrap_or(metadata.lifecycle_config_updated_at); + Some((raw_config_to_base64(&subset), axis)) + } + // Transition-only config: with an expiry axis the site once had + // expiry rules and properly removed them — the delete travels at + // that axis. Without one there is nothing to say (a delete stamped + // off the whole-config time would let a local transition edit erase + // newer peer expiry state). + None => axis.map(|axis| (None, axis)), + } +} + +/// Whether `rule` is in the shape the reconciler derives (`site-repl-` +/// naming the deployment its ARN targets). The reconciler rebuilds every such +/// rule from the current peer set — current peer or not, so a leftover from a +/// removed peer or a self-pointing rule is rebuilt away — while the merges +/// keep only the current peers' rules and treat a leftover as operator state +/// the edit replaces. An operator-authored `site-repl-*` id on an operator +/// ARN is outside the shape and survives every pass. +pub(crate) fn is_derived_site_replication_rule(rule: &ReplicationRule) -> bool { + site_replication_rule_deployment_id(rule).is_some() +} + +pub(crate) fn build_site_replication_rule(arn: &str, priority: i32, rule_id: &str) -> ReplicationRule { + ReplicationRule { + delete_marker_replication: Some(DeleteMarkerReplication { + status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)), + }), + delete_replication: Some(DeleteReplication { + status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED), + }), + destination: Destination { + bucket: arn.to_string(), + ..Default::default() + }, + existing_object_replication: Some(ExistingObjectReplication { + status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED), + }), + filter: None, + id: Some(rule_id.to_string()), + prefix: None, + priority: Some(priority), + source_selection_criteria: Some(SourceSelectionCriteria { + replica_modifications: Some(ReplicaModifications { + status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED), + }), + sse_kms_encrypted_objects: None, + }), + status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED), + } +} + +pub(crate) fn build_site_replication_config( + bucket: &str, + state: &SiteReplicationState, + local_peer: &PeerInfo, + service_account_secret_key: &str, + existing: Option<&ReplicationConfiguration>, +) -> S3Result> { + // Reuse the ARN already recorded for a peer so the rule keeps pointing at the same + // bucket target `reconcile_site_replication_bucket_targets` keys off (a MinIO-era + // `arn:minio:...` target would otherwise be orphaned by a freshly minted ARN). + let configured_arns = site_replication_target_arns_by_peer(existing); + let mut rules = Vec::new(); + for peer in state.peers.values() { + if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { + continue; + } + + let Some(target) = site_replication_bucket_target_for_peer( + bucket, + state, + peer, + service_account_secret_key, + configured_arns.get(&peer.deployment_id).cloned(), + )? + else { + continue; + }; + rules.push(build_site_replication_rule( + &target.arn, + (rules.len() + 1) as i32, + &format!("site-repl-{}", peer.deployment_id), + )); + } + + if rules.is_empty() { + Ok(None) + } else { + Ok(Some(ReplicationConfiguration { + role: String::new(), + rules, + })) + } +} + +pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime( + bucket: &str, + state: &SiteReplicationState, + local_peer: &PeerInfo, + config: Option<&ReplicationConfiguration>, + service_account_secret_key: &str, + expected_incarnation_id: Uuid, +) -> S3Result<()> { + let existing = match metadata_sys::list_bucket_targets(bucket).await { + Ok(targets) => targets, + Err(StorageError::ConfigNotFound) => BucketTargets::default(), + Err(err) => return Err(ApiError::from(err).into()), + }; + let existing_json = serde_json::to_vec(&existing) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize bucket targets failed: {e}")))?; + + let updated = + reconcile_site_replication_bucket_targets(existing, bucket, state, local_peer, config, service_account_secret_key)?; + if updated.targets.is_empty() { + return Ok(()); + } + + let json_targets = serde_json::to_vec(&updated) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize bucket targets failed: {e}")))?; + // Rewriting identical targets would churn bucket metadata and rebuild every remote S3 + // client — noticeable now that startup reconciles all buckets, not just the one bucket + // an operation touched. + if json_targets == existing_json { + return Ok(()); + } + metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + Ok(()) +} + +pub(crate) async fn bucket_replication_config_for_target_refresh(bucket: &str) -> S3Result> { + match metadata_sys::get_replication_config(bucket).await { + Ok((config, _)) => Ok(Some(config)), + Err(StorageError::ConfigNotFound) => Ok(None), + Err(err) => Err(ApiError::from(err).into()), + } +} + +pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runtime( + bucket: &str, + state: &SiteReplicationState, + local_peer: &PeerInfo, + service_account_secret_key: &str, + expected_incarnation_id: Uuid, +) -> S3Result<()> { + let existing = match metadata_sys::get_replication_config(bucket).await { + Ok((existing, _)) => Some(existing), + Err(StorageError::ConfigNotFound) => None, + Err(err) => return Err(ApiError::from(err).into()), + }; + + let Some(desired) = build_site_replication_config(bucket, state, local_peer, service_account_secret_key, existing.as_ref())? + else { + return Ok(()); + }; + + // Derived rules are state owned by this site: rebuild them from the current peer + // set on every pass instead of preserving whatever is on disk. A rule left over + // from a removed peer — or one whose destination ARN names this very deployment, + // which no bucket target can ever satisfy — must not survive, otherwise objects + // are queued against an ARN that resolves to nothing. + let (existing_role, existing_rules) = existing + .map(|config| (config.role, config.rules)) + .unwrap_or_else(|| (String::new(), Vec::new())); + let mut rules: Vec = existing_rules + .iter() + .filter(|rule| !is_derived_site_replication_rule(rule)) + .cloned() + .collect(); + rules.extend(desired.rules); + // Operator priorities are the operator's policy; only the derived rules + // take free slots, by the same function as the config merges so a merged + // write and this pass agree byte for byte. + assign_site_replication_rule_priorities(&mut rules, is_derived_site_replication_rule); + + // Only a `role` naming a current peer is ours to drop — an operator-authored role is + // part of the bucket's S3-visible configuration, and repairing a reverse rule must not + // quietly rewrite it. Same rule as `merge_incoming_replication_config`. + let role = if is_site_replication_role(&existing_role, &remote_peer_deployment_ids(state, local_peer)) { + String::new() + } else { + existing_role.clone() + }; + + if rules == existing_rules && role == existing_role { + return Ok(()); + } + + let data = serialize(&ReplicationConfiguration { role, rules }) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}")))?; + metadata_sys::update_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + + Ok(()) +} + +pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime( + bucket: &str, + runtime: &SiteReplicationRuntime, +) -> S3Result<()> { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; + ensure_site_replication_bucket_setup_with_runtime_for_incarnation(bucket, runtime, expected_incarnation_id).await +} + +pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarnation( + bucket: &str, + runtime: &SiteReplicationRuntime, + expected_incarnation_id: Uuid, +) -> S3Result<()> { + let _targets_guard = lock_bucket_targets_metadata(bucket).await; + let config = bucket_replication_config_for_target_refresh(bucket).await?; + ensure_site_replication_bucket_targets_with_runtime( + bucket, + &runtime.state, + &runtime.local_peer, + config.as_ref(), + &runtime.service_account_secret_key, + expected_incarnation_id, + ) + .await?; + ensure_site_replication_bucket_replication_config_with_runtime( + bucket, + &runtime.state, + &runtime.local_peer, + &runtime.service_account_secret_key, + expected_incarnation_id, + ) + .await?; + Ok(()) +} + +pub(crate) fn bucket_versioning_xml() -> S3Result> { + let config = VersioningConfiguration { + status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), + ..Default::default() + }; + serialize(&config).map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize versioning failed: {e}"))) +} + +pub(crate) async fn ensure_site_replication_bucket_versioning(bucket: &str) -> S3Result<()> { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; + match metadata_sys::get_versioning_config(bucket).await { + Ok((config, _)) if config.enabled() => return Ok(()), + Ok(_) | Err(StorageError::ConfigNotFound) => {} + Err(err) => return Err(ApiError::from(err).into()), + } + + metadata_sys::update_if_incarnation(bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + + Ok(()) +} diff --git a/rustfs/src/admin/site_replication_identity.rs b/rustfs/src/site_replication/identity.rs similarity index 99% rename from rustfs/src/admin/site_replication_identity.rs rename to rustfs/src/site_replication/identity.rs index 24784160b..2a4738ebf 100644 --- a/rustfs/src/admin/site_replication_identity.rs +++ b/rustfs/src/site_replication/identity.rs @@ -86,7 +86,7 @@ pub(crate) fn mark_unknown_peer_sync_enabled(peers: &mut BTreeMap bool { +pub(crate) fn is_https_endpoint(endpoint: &str) -> bool { canonical_endpoint(endpoint).starts_with("https://") } diff --git a/rustfs/src/site_replication/mod.rs b/rustfs/src/site_replication/mod.rs new file mode 100644 index 000000000..930ac8624 --- /dev/null +++ b/rustfs/src/site_replication/mod.rs @@ -0,0 +1,163 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Site-replication service subsystem (backlog#1840). +//! +//! The parts of site replication that storage-side flows call into — the +//! persisted cluster state and its RMW transaction, the peer HTTP transport, +//! the retry queue, the repair state machine, and the bucket/IAM broadcast +//! hooks — live here in the infra layer. The admin HTTP handlers stay in +//! `crate::admin::handlers::site_replication` and call down into this module; +//! that file re-exports these items so existing paths keep resolving. +//! +//! Storage access goes through the root facade (`crate::storage_api`) and +//! never through the admin or storage interface layers — this module sits +//! below the interface layer and must not import upward. + +pub(crate) mod identity; +pub(crate) mod state_lock; + +pub(crate) mod hooks; +pub(crate) mod repair; +pub(crate) mod retry; +pub(crate) mod state; +pub(crate) mod transport; + +pub(crate) use self::hooks::*; +pub(crate) use self::repair::*; +pub(crate) use self::retry::*; +pub(crate) use self::state::*; +pub(crate) use self::transport::*; + +use self::identity::{ + canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with, + same_identity_endpoint, +}; +use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock}; +use crate::auth::constant_time_eq; +use crate::config::get_config_snapshot; +use crate::error::ApiError; +use crate::runtime_sources::{ + current_deployment_id, current_endpoints_handle, current_iam_handle, current_object_store_handle, current_region, +}; +use crate::storage_api::site_replication::s3::{ + Body, BucketLifecycleConfiguration, BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, + DeleteReplication, DeleteReplicationStatus, Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, + LifecycleRule, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule, + ReplicationRuleStatus, S3Error, S3ErrorCode, S3Response, S3Result, SourceSelectionCriteria, VersioningConfiguration, + s3_error, +}; +#[cfg(test)] +use crate::storage_api::site_replication::save_config as save_admin_config; +use crate::storage_api::site_replication::{ + ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketTarget, + BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, StorageError, + VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, is_site_replication_role, + lock_bucket_targets_metadata, metadata_sys, read_config as read_admin_config, read_config_no_lock, + replication_target_arn_deployment_id, save_config_no_lock, serialize, site_replication_rule_deployment_id, + with_config_object_read_lock, with_config_object_write_lock, +}; +use base64_simd::STANDARD as BASE64_STANDARD; +use base64_simd::URL_SAFE_NO_PAD; +use hmac::{Hmac, Mac}; +use http::header::{CONTENT_TYPE, HOST}; +use http::{HeaderMap, HeaderValue, Uri}; +use hyper::{Method, StatusCode}; +use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH}; +use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type}; +use rustfs_iam::sys::SITE_REPLICATOR_SERVICE_ACCOUNT; +use rustfs_madmin::{ + AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus, SITE_REPL_API_VERSION, + SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq, SRResyncOpStatus, + SRRetryStats, SRStateInfo, SyncStatus, +}; +use rustfs_signer::constants::UNSIGNED_PAYLOAD; +use rustfs_signer::sign_v4; +use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration}; +use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url}; +use rustfs_utils::http::get_source_scheme; +use rustls_pki_types::pem::PemObject; +use serde::Deserialize; +use serde::Serialize; +use serde::de::IgnoredAny; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, LazyLock}; +use std::time::Duration; +use time::OffsetDateTime; +use tokio::sync::{Mutex, RwLock}; +use tracing::{info, warn}; +use url::{Url, form_urlencoded}; +use uuid::Uuid; + +pub(crate) const LOG_COMPONENT_ADMIN: &str = "admin"; + +pub(crate) const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication"; + +pub(crate) const EVENT_ADMIN_SITE_REPLICATION_STATE: &str = "admin_site_replication_state"; + +/// Layer-local mirror of `crate::admin::utils::json_response` (the repair +/// executor answers the admin HTTP surface but must not import upward from +/// the infra layer). +fn json_response(status: StatusCode, value: &T) -> S3Result> { + let data = serde_json::to_vec(value) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + Ok(S3Response::with_headers((status, Body::from(data)), headers)) +} + +// The admin layer's runtime-source wrappers apply fallbacks on top of +// `crate::runtime_sources`; this module reproduces the same fallbacks locally +// (verbatim from `crate::admin::runtime_sources`) so it never imports upward +// into the interface layer. + +#[cfg(test)] +static TEST_OUTBOUND_TLS_GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +#[cfg(test)] +pub(crate) fn set_test_outbound_tls_generation(generation: u64) { + crate::runtime_sources::set_test_outbound_tls_generation(generation); + TEST_OUTBOUND_TLS_GENERATION.store(generation, std::sync::atomic::Ordering::Relaxed); +} + +fn current_outbound_tls_generation() -> TlsGeneration { + crate::runtime_sources::current_outbound_tls_generation().unwrap_or_else(empty_outbound_tls_generation) +} + +#[cfg(test)] +fn empty_outbound_tls_generation() -> TlsGeneration { + TlsGeneration(TEST_OUTBOUND_TLS_GENERATION.load(std::sync::atomic::Ordering::Relaxed)) +} + +#[cfg(not(test))] +fn empty_outbound_tls_generation() -> TlsGeneration { + TlsGeneration(0) +} + +async fn current_outbound_tls_state() -> GlobalPublishedOutboundTlsState { + if let Some(state) = crate::runtime_sources::current_outbound_tls_state().await { + return state; + } + + crate::runtime_sources::fallback_outbound_tls_runtime_interface() + .state() + .await +} + +fn current_runtime_port() -> u16 { + crate::runtime_sources::current_runtime_port().unwrap_or(rustfs_config::DEFAULT_PORT) +} diff --git a/rustfs/src/site_replication/repair.rs b/rustfs/src/site_replication/repair.rs new file mode 100644 index 000000000..cecae7d61 --- /dev/null +++ b/rustfs/src/site_replication/repair.rs @@ -0,0 +1,845 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub(crate) const SITE_REPLICATION_REPAIR_STATE_PATH: &str = "config/site-replication/repair-state.json"; + +pub(crate) const SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH: &str = "config/site-replication/repair-execution.lock"; + +pub(crate) const SITE_REPLICATION_REPAIR_OPERATION_LIMIT: usize = 32; + +pub(crate) const SITE_REPLICATION_REPAIR_IAM_FAMILY: &str = "iam"; + +pub(crate) const SITE_REPLICATION_REPAIR_BUCKET_FAMILY: &str = "bucket"; + +pub(crate) const SITE_REPLICATION_REPAIR_BUCKET_METADATA_FAMILY: &str = "bucket-metadata"; + +pub(crate) const SITE_REPLICATION_REPAIR_REPLICATION_FAMILY: &str = "replication"; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairState { + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) operations: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairOperation { + pub(crate) operation_id: String, + pub(crate) preflight_token: String, + pub(crate) plan_token: String, + pub(crate) status: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) sites: BTreeMap, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) created_at: Option, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) updated_at: Option, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) completed_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairSiteStatus { + pub(crate) deployment_id: String, + pub(crate) name: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) families: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairFamilyStatus { + pub(crate) planned: usize, + pub(crate) succeeded: usize, + pub(crate) failed: usize, + #[serde(default)] + pub(crate) retry_events: usize, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) tasks: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) errors: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairTaskStatus { + pub(crate) task_id: String, + pub(crate) status: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) error: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SiteReplicationRepairRequest { + pub(crate) mode: SiteReplicationRepairMode, + #[serde(default)] + pub(crate) preflight_token: Option, + #[serde(default)] + pub(crate) operation_id: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SiteReplicationRepairMode { + DryRun, + Execute, +} + +pub(crate) struct SiteReplicationRepairExecutionRequest { + pub(crate) local_peer: PeerInfo, + pub(crate) preflight_token: String, + pub(crate) operation_id: String, + pub(crate) signing_key: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairPreflight { + pub(crate) mode: &'static str, + pub(crate) status: &'static str, + pub(crate) preflight_token: String, + pub(crate) retry_events: usize, + pub(crate) sites: BTreeMap, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairOperationResponse { + pub(crate) mode: &'static str, + pub(crate) operation_id: String, + pub(crate) status: String, + pub(crate) sites: BTreeMap, + #[serde(with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) created_at: Option, + #[serde(with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) updated_at: Option, + #[serde(with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) completed_at: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairSiteResponse { + pub(crate) deployment_id: String, + pub(crate) name: String, + pub(crate) families: BTreeMap, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SiteReplicationRepairFamilyResponse { + pub(crate) planned: usize, + pub(crate) succeeded: usize, + pub(crate) failed: usize, + pub(crate) retry_events: usize, + pub(crate) tasks: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub(crate) errors: Vec, +} + +pub(crate) async fn load_site_replication_repair_state_from_store(store: Arc) -> S3Result { + match read_config_no_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH).await { + Ok(data) => serde_json::from_slice(&data).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication repair state: {e}")) + }), + Err(StorageError::ConfigNotFound) => Ok(SiteReplicationRepairState::default()), + Err(err) => Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load site replication repair state: {err}"), + )), + } +} + +pub(crate) async fn save_site_replication_repair_state_to_store( + store: Arc, + state: &SiteReplicationRepairState, +) -> S3Result<()> { + let data = serde_json::to_vec(state) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair state failed: {e}")))?; + save_config_no_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH, data) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save repair state failed: {e}"))) +} + +pub(crate) async fn read_site_replication_repair_state() -> S3Result { + let store = + current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let read_store = store.clone(); + with_config_object_read_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH.to_string(), move || async move { + load_site_replication_repair_state_from_store(read_store).await + }) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock repair state failed: {e}")))? +} + +pub(crate) async fn update_site_replication_repair_state(update: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(&mut SiteReplicationRepairState) -> S3Result + Send + 'static, +{ + let store = + current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let read_store = store.clone(); + let save_store = store.clone(); + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH.to_string(), move || async move { + let mut state = load_site_replication_repair_state_from_store(read_store).await?; + let result = update(&mut state)?; + save_site_replication_repair_state_to_store(save_store, &state).await?; + Ok(result) + }) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock repair state failed: {e}")))? +} + +pub(crate) enum SiteReplicationRepairTask<'a> { + Iam(&'a SRIAMItem), + BucketMake(&'a str), + BucketMetadata(&'a SRBucketMeta), + Replication(&'a str), +} + +impl SiteReplicationRepairTask<'_> { + pub(crate) fn family(&self) -> &'static str { + match self { + Self::Iam(_) => SITE_REPLICATION_REPAIR_IAM_FAMILY, + Self::BucketMake(_) => SITE_REPLICATION_REPAIR_BUCKET_FAMILY, + Self::BucketMetadata(_) => SITE_REPLICATION_REPAIR_BUCKET_METADATA_FAMILY, + Self::Replication(_) => SITE_REPLICATION_REPAIR_REPLICATION_FAMILY, + } + } + + pub(crate) fn path(&self) -> &str { + match self { + Self::Iam(_) => "/rustfs/admin/v3/site-replication/peer/iam-item", + Self::BucketMake(path) | Self::Replication(path) => path, + Self::BucketMetadata(_) => "/rustfs/admin/v3/site-replication/peer/bucket-meta", + } + } + + pub(crate) fn id(&self) -> S3Result { + let payload = match self { + Self::Iam(item) => serde_json::to_vec(item), + Self::BucketMake(_) | Self::Replication(_) => serde_json::to_vec(&serde_json::json!({})), + Self::BucketMetadata(item) => serde_json::to_vec(item), + } + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair task failed: {err}")))?; + let mut digest = Sha256::new(); + digest.update(self.family().as_bytes()); + digest.update([0]); + digest.update(self.path().as_bytes()); + digest.update([0]); + digest.update(payload); + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize())) + } + + pub(crate) async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result> { + match self { + Self::Iam(item) => { + send_peer_admin_request_with_client( + &transport.client, + &transport.connection, + self.path(), + access_key, + secret_key, + item, + ) + .await + } + Self::BucketMetadata(item) => { + send_peer_admin_request_with_client( + &transport.client, + &transport.connection, + self.path(), + access_key, + secret_key, + item, + ) + .await + } + Self::BucketMake(_) | Self::Replication(_) => { + send_peer_admin_request_with_client( + &transport.client, + &transport.connection, + self.path(), + access_key, + secret_key, + &serde_json::json!({}), + ) + .await + } + } + } +} + +pub(crate) fn site_replication_repair_tasks(plan: &SiteReplicationBootstrapPlan) -> Vec<(usize, SiteReplicationRepairTask<'_>)> { + let mut tasks = Vec::with_capacity( + plan.iam_items.len() + plan.bucket_make_ops.len() + plan.bucket_items.len() + plan.bucket_configure_ops.len(), + ); + tasks.extend( + plan.iam_items + .iter() + .enumerate() + .map(|(index, item)| (index, SiteReplicationRepairTask::Iam(item))), + ); + tasks.extend( + plan.bucket_make_ops + .iter() + .enumerate() + .map(|(index, path)| (index, SiteReplicationRepairTask::BucketMake(path))), + ); + tasks.extend( + plan.bucket_items + .iter() + .enumerate() + .map(|(index, item)| (index, SiteReplicationRepairTask::BucketMetadata(item))), + ); + tasks.extend( + plan.bucket_configure_ops + .iter() + .enumerate() + .map(|(index, path)| (index, SiteReplicationRepairTask::Replication(path))), + ); + tasks +} + +pub(crate) fn site_replication_repair_plan_token( + state: &SiteReplicationState, + plan: &SiteReplicationBootstrapPlan, +) -> S3Result { + let mut digest = Sha256::new(); + let snapshot = serde_json::to_vec(&( + &state.name, + &state.service_account_access_key, + &state.peers, + state.updated_at, + state.sync_state_initialized, + )) + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair snapshot failed: {err}")))?; + digest.update(snapshot); + for (_, task) in site_replication_repair_tasks(plan) { + digest.update(task.id()?.as_bytes()); + } + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize())) +} + +pub(crate) fn site_replication_repair_preflight_token( + state: &SiteReplicationState, + plan: &SiteReplicationBootstrapPlan, + signing_key: &[u8], +) -> S3Result { + if signing_key.is_empty() { + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "repair signing key is empty".to_string(), + )); + } + let mut digest = as hmac::digest::KeyInit>::new_from_slice(signing_key) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid repair signing key".to_string()))?; + digest.update(b"rustfs:site-replication:repair-preflight:v1\0"); + digest.update(site_replication_repair_plan_token(state, plan)?.as_bytes()); + for event in state + .retry_queue + .iter() + .filter(|event| retry_event_replayed_by_bootstrap(event)) + { + digest.update(event.id.as_bytes()); + digest.update(&[0]); + digest.update(event.peer_deployment_id.as_bytes()); + digest.update(&[0]); + digest.update(event.path.as_bytes()); + digest.update(&[0]); + } + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes())) +} + +pub(crate) fn site_replication_repair_task_checkpoint_id( + signing_key: &[u8], + peer_deployment_id: &str, + task: &SiteReplicationRepairTask<'_>, +) -> S3Result { + let mut digest = as hmac::digest::KeyInit>::new_from_slice(signing_key) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid repair signing key".to_string()))?; + digest.update(b"rustfs:site-replication:repair-task:v1\0"); + digest.update(peer_deployment_id.as_bytes()); + digest.update(&[0]); + digest.update(task.id()?.as_bytes()); + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes())) +} + +pub(crate) fn site_replication_repair_sites( + state: &SiteReplicationState, + local_peer: &PeerInfo, + plan: &SiteReplicationBootstrapPlan, + signing_key: &[u8], +) -> S3Result> { + let mut planned = BTreeMap::new(); + let mut family_paths = BTreeMap::>::new(); + for (_, task) in site_replication_repair_tasks(plan) { + let family = task.family().to_string(); + let family_status = planned + .entry(task.family().to_string()) + .or_insert_with(SiteReplicationRepairFamilyStatus::default); + family_status.planned += 1; + family_paths.entry(family).or_default().insert(task.path().to_string()); + } + + let mut sites = BTreeMap::new(); + for peer in state.peers.values().filter(|peer| { + peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + }) { + let mut families = planned.clone(); + for (_, task) in site_replication_repair_tasks(plan) { + let family = families + .get_mut(task.family()) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair task family is missing".to_string()))?; + family.tasks.push(SiteReplicationRepairTaskStatus { + task_id: site_replication_repair_task_checkpoint_id(signing_key, &peer.deployment_id, &task)?, + status: "planned".to_string(), + error: None, + }); + } + for (family, status) in &mut families { + status.retry_events = state + .retry_queue + .iter() + .filter(|event| { + event.peer_deployment_id == peer.deployment_id + && retry_event_replayed_by_bootstrap(event) + && family_paths.get(family).is_some_and(|paths| paths.contains(&event.path)) + }) + .count(); + } + sites.insert( + peer.deployment_id.clone(), + SiteReplicationRepairSiteStatus { + deployment_id: peer.deployment_id.clone(), + name: peer.name.clone(), + families, + }, + ); + } + Ok(sites) +} + +pub(crate) fn update_site_replication_repair_task( + operation: &mut SiteReplicationRepairOperation, + deployment_id: &str, + family: &str, + family_index: usize, + result: Result<(), &str>, +) -> S3Result<()> { + let site = operation + .sites + .get_mut(deployment_id) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation site is missing".to_string()))?; + let family_status = site + .families + .get_mut(family) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation family is missing".to_string()))?; + if family_status.succeeded != family_index { + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "repair operation task checkpoint is invalid".to_string(), + )); + } + let task_status = family_status.tasks.get_mut(family_index).ok_or_else(|| { + S3Error::with_message(S3ErrorCode::InternalError, "repair operation task checkpoint is missing".to_string()) + })?; + family_status.failed = 0; + family_status.errors.clear(); + match result { + Ok(()) => { + family_status.succeeded = family_status.succeeded.saturating_add(1); + task_status.status = "succeeded".to_string(); + task_status.error = None; + } + Err(error) => { + let error = classify_site_replication_repair_error(error).to_string(); + family_status.failed = 1; + family_status.errors.push(error.clone()); + task_status.status = "failed".to_string(); + task_status.error = Some(error); + } + } + Ok(()) +} + +pub(crate) fn site_replication_repair_task_pending( + operation: &SiteReplicationRepairOperation, + deployment_id: &str, + family: &str, + family_index: usize, +) -> S3Result { + let site = operation + .sites + .get(deployment_id) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation site is missing".to_string()))?; + let family = site + .families + .get(family) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair operation family is missing".to_string()))?; + if family.succeeded > family_index { + return Ok(false); + } + if family.succeeded < family_index { + return Ok(false); + } + Ok(family.failed == 0) +} + +pub(crate) fn prepare_site_replication_repair_retry(operation: &mut SiteReplicationRepairOperation) { + for family in operation.sites.values_mut().flat_map(|site| site.families.values_mut()) { + family.failed = 0; + family.errors.clear(); + for task in &mut family.tasks { + match task.status.as_str() { + "succeeded" => task.status = "skipped".to_string(), + "failed" => { + task.status = "planned".to_string(); + task.error = None; + } + _ => {} + } + } + } +} + +pub(crate) fn classify_site_replication_repair_error(error: &str) -> &'static str { + let error = error.to_ascii_lowercase(); + if error.contains("accessdenied") + || error.contains("signaturedoesnotmatch") + || error.contains("unauthorized") + || error.contains("forbidden") + || error.contains("401") + || error.contains("403") + { + "authorization-failed" + } else if error.contains("timeout") { + "remote-timeout" + } else if error.contains("dns") { + "remote-dns-failed" + } else if error.contains("tls") || error.contains("certificate") { + "remote-tls-failed" + } else if error.contains("connect") { + "remote-connect-failed" + } else { + "remote-operation-failed" + } +} + +pub(crate) fn summarize_site_replication_repair_operation(operation: &mut SiteReplicationRepairOperation) { + let failed = operation + .sites + .values() + .flat_map(|site| site.families.values()) + .any(|family| family.failed > 0); + let complete = operation + .sites + .values() + .all(|site| site.families.values().all(|family| family.succeeded == family.planned)); + operation.status = if complete { + "success" + } else if failed { + "partial" + } else { + "running" + } + .to_string(); + operation.updated_at = Some(OffsetDateTime::now_utc()); + operation.completed_at = complete.then_some(OffsetDateTime::now_utc()); +} + +pub(crate) fn site_replication_repair_operation_response( + operation: &SiteReplicationRepairOperation, +) -> SiteReplicationRepairOperationResponse { + SiteReplicationRepairOperationResponse { + mode: "execute", + operation_id: operation.operation_id.clone(), + status: operation.status.clone(), + sites: operation + .sites + .iter() + .map(|(deployment_id, site)| { + ( + deployment_id.clone(), + SiteReplicationRepairSiteResponse { + deployment_id: site.deployment_id.clone(), + name: site.name.clone(), + families: site + .families + .iter() + .map(|(family, status)| { + ( + family.clone(), + SiteReplicationRepairFamilyResponse { + planned: status.planned, + succeeded: status.succeeded, + failed: status.failed, + retry_events: status.retry_events, + tasks: status.tasks.clone(), + errors: status.errors.clone(), + }, + ) + }) + .collect(), + }, + ) + }) + .collect(), + created_at: operation.created_at, + updated_at: operation.updated_at, + completed_at: operation.completed_at, + } +} + +pub(crate) fn prune_site_replication_repair_operations(operations: &mut BTreeMap) { + while operations.len() > SITE_REPLICATION_REPAIR_OPERATION_LIMIT { + let Some(oldest) = operations + .iter() + .filter(|(_, operation)| operation.status == "success") + .min_by_key(|(_, operation)| operation.created_at) + .map(|(id, _)| id.clone()) + else { + break; + }; + operations.remove(&oldest); + } +} + +pub(crate) async fn persist_site_replication_repair_operation(operation: &SiteReplicationRepairOperation) -> S3Result<()> { + let operation = operation.clone(); + update_site_replication_repair_state(move |state| { + if let Some(existing) = state.operations.get(&operation.operation_id) + && !constant_time_eq(&existing.preflight_token, &operation.preflight_token) + { + return Err(S3Error::with_message( + S3ErrorCode::ClientTokenConflict, + "repair operation ID is already bound to a different preflight".to_string(), + )); + } + state.operations.insert(operation.operation_id.clone(), operation); + prune_site_replication_repair_operations(&mut state.operations); + Ok(()) + }) + .await +} + +pub(crate) async fn persist_site_replication_repair_task( + operation: &SiteReplicationRepairOperation, + peer: &PeerInfo, + family: &str, + path: &str, +) -> S3Result<()> { + persist_site_replication_repair_operation(operation).await?; + + let family_status = operation + .sites + .get(&peer.deployment_id) + .and_then(|site| site.families.get(family)) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair task status is missing".to_string()))?; + let failure = (family_status.failed > 0).then(|| { + family_status + .errors + .first() + .cloned() + .unwrap_or_else(|| "remote-operation-failed".to_string()) + }); + let peer = peer.clone(); + let path = path.to_string(); + update_site_replication_state(move |state| { + match failure.as_deref() { + Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), + None => { + dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); + } + } + Ok(()) + }) + .await +} + +pub(crate) fn admit_site_replication_repair_operation( + repair_state: &mut SiteReplicationRepairState, + operation_id: String, + supplied_token: &str, + candidate: SiteReplicationRepairOperation, +) -> S3Result { + if let Some(existing) = repair_state.operations.get(&operation_id) { + if !constant_time_eq(&existing.preflight_token, supplied_token) { + return Err(S3Error::with_message( + S3ErrorCode::ClientTokenConflict, + "repair operation ID is already bound to a different preflight".to_string(), + )); + } + if !constant_time_eq(&existing.plan_token, &candidate.plan_token) { + return Err(S3Error::with_message( + S3ErrorCode::PreconditionFailed, + "site replication repair plan changed after partial execution".to_string(), + )); + } + return Ok(existing.clone()); + } + if repair_state + .operations + .values() + .any(|operation| operation.status == "running") + { + return Err(S3Error::with_message( + S3ErrorCode::ClientTokenConflict, + "another site replication repair is active".to_string(), + )); + } + repair_state.operations.insert(operation_id, candidate.clone()); + prune_site_replication_repair_operations(&mut repair_state.operations); + Ok(candidate) +} + +pub(crate) async fn execute_site_replication_repair( + request: SiteReplicationRepairExecutionRequest, +) -> S3Result> { + let store = + current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + execute_site_replication_repair_locked(request).await + }) + .await + .map_err(|_| { + S3Error::with_message(S3ErrorCode::ClientTokenConflict, "another site replication repair is active".to_string()) + })? +} + +pub(crate) async fn execute_site_replication_repair_locked( + request: SiteReplicationRepairExecutionRequest, +) -> S3Result> { + let state = load_site_replication_state().await?; + if !state.enabled() || state.service_account_access_key.is_empty() { + return Err(s3_error!(InvalidRequest, "site replication is not configured")); + } + let info = build_sr_info(&state, &request.local_peer).await?; + let plan = site_replication_bootstrap_plan(&info)?; + let plan_token = site_replication_repair_plan_token(&state, &plan)?; + let preflight_token = site_replication_repair_preflight_token(&state, &plan, request.signing_key.as_bytes())?; + let sites = site_replication_repair_sites(&state, &request.local_peer, &plan, request.signing_key.as_bytes())?; + + let repair_state = read_site_replication_repair_state().await?; + if let Some(existing) = repair_state.operations.get(&request.operation_id) { + if !constant_time_eq(&existing.preflight_token, &request.preflight_token) { + return Err(S3Error::with_message( + S3ErrorCode::ClientTokenConflict, + "repair operation ID is already bound to a different preflight".to_string(), + )); + } + if existing.status == "success" { + return json_response(StatusCode::OK, &site_replication_repair_operation_response(existing)); + } + if !constant_time_eq(&existing.plan_token, &plan_token) { + return Err(S3Error::with_message( + S3ErrorCode::PreconditionFailed, + "site replication repair plan changed after partial execution".to_string(), + )); + } + } else if !constant_time_eq(&request.preflight_token, &preflight_token) { + return Err(S3Error::with_message( + S3ErrorCode::PreconditionFailed, + "site replication repair preflight is stale".to_string(), + )); + } + + let now = OffsetDateTime::now_utc(); + let candidate = SiteReplicationRepairOperation { + operation_id: request.operation_id.clone(), + preflight_token, + plan_token, + status: "running".to_string(), + sites, + created_at: Some(now), + updated_at: Some(now), + completed_at: None, + }; + let supplied_token = request.preflight_token; + let operation_id = request.operation_id; + let mut operation = update_site_replication_repair_state(move |repair_state| { + admit_site_replication_repair_operation(repair_state, operation_id, &supplied_token, candidate) + }) + .await?; + if operation.status == "success" { + return json_response(StatusCode::OK, &site_replication_repair_operation_response(&operation)); + } + + let service_account_secret_key = site_replicator_service_account_secret(&state.service_account_access_key).await?; + prepare_site_replication_repair_retry(&mut operation); + operation.status = "running".to_string(); + operation.completed_at = None; + operation.updated_at = Some(OffsetDateTime::now_utc()); + persist_site_replication_repair_operation(&operation).await?; + + let tasks = site_replication_repair_tasks(&plan); + for peer in state.peers.values().filter(|peer| { + peer.deployment_id != request.local_peer.deployment_id + && !same_identity_endpoint(&peer.endpoint, &request.local_peer.endpoint) + }) { + let transport = match PeerTransport::for_runtime_peer(peer).await { + Ok(transport) => transport, + Err(err) => { + let error = err.to_string(); + for (family_index, task) in &tasks { + if !site_replication_repair_task_pending(&operation, &peer.deployment_id, task.family(), *family_index)? { + continue; + } + update_site_replication_repair_task( + &mut operation, + &peer.deployment_id, + task.family(), + *family_index, + Err(&error), + )?; + summarize_site_replication_repair_operation(&mut operation); + persist_site_replication_repair_task(&operation, peer, task.family(), task.path()).await?; + } + continue; + } + }; + + for (family_index, task) in &tasks { + if !site_replication_repair_task_pending(&operation, &peer.deployment_id, task.family(), *family_index)? { + continue; + } + let result = task + .send(&transport, &state.service_account_access_key, &service_account_secret_key) + .await; + let error = result.err().map(|err| err.to_string()); + update_site_replication_repair_task( + &mut operation, + &peer.deployment_id, + task.family(), + *family_index, + match error.as_deref() { + Some(error) => Err(error), + None => Ok(()), + }, + )?; + summarize_site_replication_repair_operation(&mut operation); + persist_site_replication_repair_task(&operation, peer, task.family(), task.path()).await?; + } + } + + summarize_site_replication_repair_operation(&mut operation); + persist_site_replication_repair_operation(&operation).await?; + json_response(StatusCode::OK, &site_replication_repair_operation_response(&operation)) +} diff --git a/rustfs/src/site_replication/retry.rs b/rustfs/src/site_replication/retry.rs new file mode 100644 index 000000000..0a0a81c50 --- /dev/null +++ b/rustfs/src/site_replication/retry.rs @@ -0,0 +1,928 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub(crate) const SITE_REPLICATION_RETRY_QUEUE_LIMIT: usize = 256; + +pub(crate) const SITE_REPLICATION_RETRY_FAILED_AFTER: u32 = 3; + +pub(crate) const SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH: &str = "internal:endpoint-target-refresh"; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct SiteReplicationRetryEvent { + pub(crate) id: String, + pub(crate) peer_deployment_id: String, + pub(crate) peer_endpoint: String, + pub(crate) path: String, + pub(crate) retry_count: u32, + pub(crate) failed: bool, + pub(crate) last_error: String, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) updated_at: Option, + /// Peer-edit generation whose delivery failed, when the failing send + /// carried one. Settling a *later* success for the same (peer, path) must + /// not erase a failure recorded for a NEWER generation — see + /// [`settle_site_replication_retry_events`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) edit_generation: Option, +} + +pub(crate) fn retry_event_matches(event: &SiteReplicationRetryEvent, peer: &PeerInfo, path: &str) -> bool { + (event.peer_deployment_id == peer.deployment_id || event.peer_endpoint == peer.endpoint) && event.path == path +} + +pub(crate) const SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH: &str = "internal:retry-snapshot:iam"; + +pub(crate) const SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH: &str = "internal:retry-snapshot:bucket-metadata"; + +pub(crate) fn collapsed_retry_queue_path(path: &str) -> Option<&'static str> { + let base_path = path.split_once('?').map(|(base, _)| base).unwrap_or(path); + match base_path { + "/rustfs/admin/v3/site-replication/peer/iam-item" | SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => { + Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) + } + "/rustfs/admin/v3/site-replication/peer/bucket-meta" | SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => { + Some(SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH) + } + _ => None, + } +} + +pub(crate) fn normalize_collapsed_retry_queue_paths(queue: &mut Vec) -> bool { + let mut changed = false; + let mut normalized: Vec = Vec::with_capacity(queue.len()); + for mut event in queue.drain(..) { + if let Some(path) = collapsed_retry_queue_path(&event.path) + && event.path != path + { + event.path = path.to_string(); + changed = true; + } + + let duplicate = normalized.iter().position(|existing| { + existing.path == event.path + && (existing.peer_deployment_id == event.peer_deployment_id || existing.peer_endpoint == event.peer_endpoint) + }); + let Some(index) = duplicate else { + normalized.push(event); + continue; + }; + + changed = true; + let existing = &mut normalized[index]; + let event_is_newer = match (event.updated_at, existing.updated_at) { + (Some(event), Some(existing)) => event >= existing, + (Some(_), None) => true, + _ => false, + }; + if event_is_newer { + let retry_count = existing.retry_count.max(event.retry_count); + *existing = event; + existing.retry_count = retry_count; + } else { + existing.retry_count = existing.retry_count.max(event.retry_count); + } + existing.failed = existing.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; + } + *queue = normalized; + changed +} + +pub(crate) async fn migrate_collapsed_retry_queue_paths() -> S3Result<()> { + update_site_replication_state_when_changed(|state| { + Ok(if normalize_collapsed_retry_queue_paths(&mut state.retry_queue) { + StateCommit::Changed(()) + } else { + StateCommit::Unchanged(()) + }) + }) + .await +} + +#[cfg(test)] +pub(crate) fn dequeue_site_replication_retry_events( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, +) -> usize { + settle_site_replication_retry_events(queue, peer, path, None) +} + +/// Repair-path settlement: also clears snapshot-escalated entries. Running a +/// repair is the operator's explicit accountability transfer for the +/// possibly-unreplayed deletion the marker records; ordinary delivery +/// successes must not clear it (see [`settle_site_replication_retry_events`]). +pub(crate) fn dequeue_site_replication_retry_events_including_escalated( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, +) -> usize { + let before = queue.len(); + let collapsed_path = collapsed_retry_queue_path(path); + queue.retain(|event| { + !retry_event_matches(event, peer, path) + && !collapsed_path.is_some_and(|collapsed_path| retry_event_matches(event, peer, collapsed_path)) + }); + before.saturating_sub(queue.len()) +} + +/// Remove the retry events for (peer, path) that `generation` is entitled to +/// settle. A successful delivery only proves the peer reached the state the +/// delivery carried: while it was in flight another edit can commit, fail its +/// own delivery, and enqueue for the same (peer, path). Erasing that event +/// would leave the peer on the older edit with no retry left, so an event +/// stamped with a NEWER generation survives. `None` settles unconditionally — +/// the broadcast paths that carry no generation, whose retry events live under +/// their own paths and never collide with peer-edit deliveries. +pub(crate) fn settle_site_replication_retry_events( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, + generation: Option, +) -> usize { + let before = queue.len(); + let collapsed_path = collapsed_retry_queue_path(path); + queue.retain(|event| { + if !retry_event_matches(event, peer, path) { + return true; + } + // A wire-path success identifies no IAM or bucket-metadata entity. + // This also protects legacy rows until the startup migration moves + // them under their internal snapshot path. + if collapsed_path.is_some() { + return true; + } + // A snapshot-escalated entry records a possibly-unreplayed deletion. + // Collapsed paths are shared by every entity, so a later successful + // delivery of a DIFFERENT item proves nothing about the deleted one — + // only a repair settles it (dequeue_..._including_escalated). + if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return true; + } + match (generation, event.edit_generation) { + (Some(settled), Some(failed)) => failed > settled, + _ => false, + } + }); + before.saturating_sub(queue.len()) +} + +pub(crate) fn upsert_site_replication_retry_event( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, + error: &str, + generation: Option, +) { + let path = collapsed_retry_queue_path(path).unwrap_or(path); + let now = OffsetDateTime::now_utc(); + let detail = summarize_peer_error_detail(error); + if let Some(event) = queue.iter_mut().find(|event| retry_event_matches(event, peer, path)) { + event.retry_count = event.retry_count.saturating_add(1); + event.failed = event.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; + event.last_error = detail; + event.updated_at = Some(now); + // Keep the newest generation: an older delivery that fails afterwards + // must not lower the fence and let its own success settle the event. + event.edit_generation = event.edit_generation.max(generation); + return; + } + + queue.push(SiteReplicationRetryEvent { + id: Uuid::new_v4().to_string(), + peer_deployment_id: peer.deployment_id.clone(), + peer_endpoint: peer.endpoint.clone(), + path: path.to_string(), + retry_count: 1, + failed: false, + last_error: detail, + updated_at: Some(now), + edit_generation: generation, + }); + if queue.len() > SITE_REPLICATION_RETRY_QUEUE_LIMIT { + let overflow = queue.len() - SITE_REPLICATION_RETRY_QUEUE_LIMIT; + queue.drain(0..overflow); + } +} + +pub(crate) fn retry_stats_for_state(state: &SiteReplicationState) -> Option { + if state.retry_queue.is_empty() { + return None; + } + + Some(SRRetryStats { + pending: state.retry_queue.iter().filter(|event| !event.failed).count(), + failed: state.retry_queue.iter().filter(|event| event.failed).count(), + last_error: state + .retry_queue + .iter() + .rev() + .find_map(|event| (!event.last_error.is_empty()).then(|| event.last_error.clone())) + .unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }) +} + +pub(crate) async fn enqueue_site_replication_retry_event(peer: &PeerInfo, path: &str, error: &S3Error) { + enqueue_site_replication_retry_event_for_generation(peer, path, error, None).await +} + +pub(crate) async fn enqueue_site_replication_retry_event_for_generation( + peer: &PeerInfo, + path: &str, + error: &S3Error, + generation: Option, +) { + let peer_owned = peer.clone(); + let path_owned = path.to_string(); + let error_text = error.to_string(); + let result = update_site_replication_state(move |state| { + // A peer that left the state can never drain its entries again + // (remove_sites already pruned them); recording a late failure for it + // would only pollute retry_stats until the queue cap evicts it. + if state.peers.contains_key(&peer_owned.deployment_id) { + upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + } + Ok(()) + }) + .await; + + if let Err(err) = result { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + peer = %peer.endpoint, + path, + error = ?err, + "failed to persist site replication retry event" + ); + } +} + +pub(crate) fn retry_bucket_operation(path: &str) -> Option { + let (base_path, query) = path.split_once('?')?; + if base_path != SITE_REPLICATION_PEER_BUCKET_OPS_PATH { + return None; + } + + form_urlencoded::parse(query.as_bytes()).find_map(|(key, value)| (key == "operation").then(|| value.into_owned())) +} + +pub(crate) fn retry_event_replayed_by_bootstrap(event: &SiteReplicationRetryEvent) -> bool { + matches!( + retry_bucket_operation(&event.path).as_deref(), + Some(SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING | SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION) + ) +} + +/// Exponential backoff base for the background retry drain, aligned with the +/// reconcile cadence (`site_replication_reconcile::RECONCILE_INTERVAL`). +pub(crate) const SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS: i64 = 600; + +/// Backoff ceiling: a permanently failed peer is still probed daily. +pub(crate) const SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS: i64 = 86_400; + +/// What the background drain may do for one retry event. Everything not +/// representable here is operator territory (manual repair). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum RetryDrainAction { + /// Constant-path IAM item deliveries collapse into one queue entry per + /// peer and their bodies are not persisted; the only faithful replay is + /// the current IAM snapshot from the bootstrap plan. + IamSnapshot, + /// Same collapse for bucket-meta deliveries: replay the bucket metadata + /// snapshot from the bootstrap plan. + BucketMetadataSnapshot, + /// A self-contained bucket op the bootstrap plan can re-derive for its + /// bucket (`make-with-versioning` / `configure-replication`). + BucketOpReplay { operation: String, bucket: String }, + /// Re-send the current peer records under a fresh edit generation. + PeerEdit, +} + +#[derive(Clone)] +pub(crate) enum RetrySnapshot { + Iam(Vec), + BucketMetadata(Vec), +} + +impl RetrySnapshot { + pub(crate) fn from_plan(action: &RetryDrainAction, plan: &SiteReplicationBootstrapPlan) -> Option { + match action { + RetryDrainAction::IamSnapshot => Some(Self::Iam(plan.iam_items.clone())), + RetryDrainAction::BucketMetadataSnapshot => Some(Self::BucketMetadata(plan.bucket_items.clone())), + _ => None, + } + } + + pub(crate) fn fingerprint(&self) -> S3Result>> { + let mut payloads = match self { + Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + } + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?; + payloads.sort_unstable(); + Ok(payloads) + } + + pub(crate) fn replay_after_change(previous: &Self, fresh: &Self, observed_at: OffsetDateTime) -> Self { + match (previous, fresh) { + (Self::Iam(previous), Self::Iam(fresh)) => { + let fresh_keys: HashSet = fresh.iter().filter_map(iam_snapshot_key).collect(); + let mut replay = fresh.clone(); + for item in previous { + if iam_snapshot_key(item).is_some_and(|key| !fresh_keys.contains(&key)) { + replay.extend(iam_snapshot_tombstones(item, observed_at)); + } + } + Self::Iam(replay) + } + (Self::BucketMetadata(previous), Self::BucketMetadata(fresh)) => { + let fresh_keys: HashSet<(&str, &str)> = fresh + .iter() + .map(|item| (item.bucket.as_str(), item.r#type.as_str())) + .collect(); + let mut replay = fresh.clone(); + for item in previous { + if !fresh_keys.contains(&(item.bucket.as_str(), item.r#type.as_str())) { + replay.push(bucket_metadata_snapshot_tombstone(item, observed_at)); + } + } + Self::BucketMetadata(replay) + } + _ => fresh.clone(), + } + } + + pub(crate) async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<()> { + match self { + Self::Iam(items) => { + for item in items { + SiteReplicationRepairTask::Iam(item) + .send(transport, access_key, secret_key) + .await?; + } + } + Self::BucketMetadata(items) => { + for item in items { + SiteReplicationRepairTask::BucketMetadata(item) + .send(transport, access_key, secret_key) + .await?; + } + } + } + Ok(()) + } +} + +#[derive(Hash, PartialEq, Eq)] +pub(crate) enum IamSnapshotKey { + Policy(String), + User(String), + Group(String), + PolicyMapping { target: String, user_type: i64, is_group: bool }, +} + +pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option { + match item.r#type.as_str() { + "policy" => Some(IamSnapshotKey::Policy(item.name.clone())), + "iam-user" => item + .iam_user + .as_ref() + .map(|user| IamSnapshotKey::User(user.access_key.clone())), + "group-info" => item + .group_info + .as_ref() + .map(|group| IamSnapshotKey::Group(group.update_req.group.clone())), + "policy-mapping" => item.policy_mapping.as_ref().map(|mapping| IamSnapshotKey::PolicyMapping { + target: mapping.user_or_group.clone(), + user_type: mapping.user_type, + is_group: mapping.is_group, + }), + _ => None, + } +} + +pub(crate) fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateTime) -> Vec { + let mut tombstone = item.clone(); + tombstone.updated_at = Some(observed_at); + match item.r#type.as_str() { + "policy" => tombstone.policy = None, + "iam-user" => { + if let Some(user) = tombstone.iam_user.as_mut() { + user.is_delete_req = true; + user.user_req = None; + } + } + "group-info" => { + let Some(group) = tombstone.group_info.as_mut() else { + return Vec::new(); + }; + group.update_req.is_remove = true; + if group.update_req.members.is_empty() { + return vec![tombstone]; + } + let mut delete = tombstone.clone(); + if let Some(group) = delete.group_info.as_mut() { + group.update_req.members.clear(); + } + return vec![tombstone, delete]; + } + "policy-mapping" => { + if let Some(mapping) = tombstone.policy_mapping.as_mut() { + mapping.policy.clear(); + } + } + _ => return Vec::new(), + } + vec![tombstone] +} + +pub(crate) fn bucket_metadata_snapshot_tombstone(item: &SRBucketMeta, observed_at: OffsetDateTime) -> SRBucketMeta { + SRBucketMeta { + r#type: item.r#type.clone(), + bucket: item.bucket.clone(), + updated_at: Some(observed_at), + expiry_updated_at: Some(observed_at), + api_version: item.api_version.clone(), + derived_rule_contract: item.derived_rule_contract, + ..Default::default() + } +} + +pub(crate) const SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS: usize = 3; + +pub(crate) fn classify_site_replication_retry_event(event: &SiteReplicationRetryEvent) -> Option { + let snapshot_action = match event.path.as_str() { + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => Some(RetryDrainAction::IamSnapshot), + SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => Some(RetryDrainAction::BucketMetadataSnapshot), + _ => None, + }; + if snapshot_action.is_some() && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return snapshot_action; + } + if event.path.starts_with("internal:") { + // Marker records store payloads in `last_error` (legacy + // pending-endpoint-refresh backup and snapshot liabilities); they are + // not drainable delivery failures. + return None; + } + if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + // Already snapshot-replayed once for this failure episode; a possible + // deletion cannot be replayed from a snapshot, so re-sending daily + // proves nothing. A new hook failure overwrites the marker. + return None; + } + let base_path = event.path.split_once('?').map(|(base, _)| base).unwrap_or(&event.path); + match base_path { + "/rustfs/admin/v3/site-replication/peer/iam-item" => Some(RetryDrainAction::IamSnapshot), + "/rustfs/admin/v3/site-replication/peer/bucket-meta" => Some(RetryDrainAction::BucketMetadataSnapshot), + SITE_REPLICATION_PEER_EDIT_PATH => Some(RetryDrainAction::PeerEdit), + SITE_REPLICATION_PEER_BUCKET_OPS_PATH => { + let operation = retry_bucket_operation(&event.path)?; + if !matches!( + operation.as_str(), + SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING | SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION + ) { + // Destructive ops (delete-bucket / force-delete-bucket) are + // operator territory: replaying them against a peer whose + // bucket was since recreated is irreversible. + return None; + } + let bucket = retry_bucket_name(&event.path)?; + Some(RetryDrainAction::BucketOpReplay { operation, bucket }) + } + _ => None, + } +} + +pub(crate) fn retry_bucket_name(path: &str) -> Option { + let (_, query) = path.split_once('?')?; + form_urlencoded::parse(query.as_bytes()) + .find_map(|(key, value)| (key == "bucket" && !value.is_empty()).then(|| value.into_owned())) +} + +/// A collapsed retry event after a stable snapshot resend is escalated with +/// this marker instead of being cleared: the snapshot contains no task for a +/// failed deletion, so remote absence remains operator-visible. Collapsed +/// failures use an internal queue path so ordinary successes and older nodes +/// cannot settle an unrelated entity's liability. +pub(crate) const SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER: &str = "snapshot replayed; a failed deletion cannot be replayed from a snapshot — run site replication repair or re-deliver to settle"; + +/// Escalate a collapsed retry event after its snapshot resend succeeded, +/// unless a newer failure was recorded after `snapshot_updated_at` (that +/// failure belongs to a newer local commit the snapshot did not contain and +/// must keep the entry drain-eligible). +pub(crate) fn escalate_site_replication_retry_events_up_to( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, + snapshot_updated_at: Option, +) -> usize { + let Some(marker_path) = collapsed_retry_queue_path(path) else { + return 0; + }; + + if path != marker_path { + queue.retain(|event| { + if !retry_event_matches(event, peer, path) { + return true; + } + matches!((event.updated_at, snapshot_updated_at), (Some(current), Some(seen)) if current > seen) + || matches!((event.updated_at, snapshot_updated_at), (Some(_), None)) + }); + } + + let marker_index = queue.iter().position(|event| retry_event_matches(event, peer, marker_path)); + let marker_index = marker_index.unwrap_or_else(|| { + queue.push(SiteReplicationRetryEvent { + id: Uuid::new_v4().to_string(), + peer_deployment_id: peer.deployment_id.clone(), + peer_endpoint: peer.endpoint.clone(), + path: marker_path.to_string(), + updated_at: snapshot_updated_at, + ..Default::default() + }); + queue.len() - 1 + }); + let event = &mut queue[marker_index]; + let newer_failure_recorded = match (event.updated_at, snapshot_updated_at) { + (Some(current), Some(seen)) => current > seen, + (Some(_), None) => true, + (None, _) => false, + }; + if newer_failure_recorded && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return 0; + } + event.failed = true; + event.retry_count = event.retry_count.max(SITE_REPLICATION_RETRY_FAILED_AFTER); + event.last_error = SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER.to_string(); + event.updated_at = Some(OffsetDateTime::now_utc()); + 1 +} + +pub(crate) async fn escalate_site_replication_retry_event_up_to( + peer: &PeerInfo, + path: &str, + snapshot_updated_at: Option, +) { + let peer_owned = peer.clone(); + let path_owned = path.to_string(); + let result = update_site_replication_state(move |state| { + escalate_site_replication_retry_events_up_to(&mut state.retry_queue, &peer_owned, &path_owned, snapshot_updated_at); + Ok(()) + }) + .await; + + if let Err(err) = result { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + peer = %peer.endpoint, + deployment_id = %peer.deployment_id, + path, + error = ?err, + "failed to escalate site replication retry event" + ); + } +} + +/// Whether the drain may attempt this event now. +pub(crate) fn site_replication_retry_backoff_elapsed(event: &SiteReplicationRetryEvent, now: OffsetDateTime) -> bool { + let Some(updated_at) = event.updated_at else { + return true; + }; + // 600 * 2^8 already exceeds the daily ceiling; capping the shift keeps + // the arithmetic overflow-free for any persisted retry_count. + let exponent = event.retry_count.saturating_sub(1).min(8); + let delay = (SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS << exponent).min(SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS); + now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= delay +} + +/// The subset of the retry queue the background drain is allowed to touch. +pub(crate) fn actionable_site_replication_retry_events( + state: &SiteReplicationState, + now: OffsetDateTime, +) -> Vec { + state + .retry_queue + .iter() + .filter(|event| classify_site_replication_retry_event(event).is_some()) + .filter(|event| state.peers.contains_key(&event.peer_deployment_id)) + .filter(|event| site_replication_retry_backoff_elapsed(event, now)) + .cloned() + .collect() +} + +/// Background consumer for the retry queue, run from the reconcile tick. +/// +/// Scope: this settles "delivered once and failed" entries whose replay is +/// faithful (bucket ops, peer edits). Collapsed iam-item / bucket-meta +/// entries are snapshot-resent and then *escalated*, not cleared — a failed +/// deletion leaves no task in the snapshot, so remote absence stays unproven +/// until a later delivery or a manual repair. A hook that never fired (crash +/// between the local commit and the send) leaves no entry at all, so the +/// drain is not a full cross-site diff-heal; manual repair remains the +/// authoritative catch-all. +pub(crate) async fn drain_site_replication_retry_queue() { + if let Err(err) = drain_site_replication_retry_queue_inner().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_failed", + error = ?err, + "admin site replication state" + ); + } +} + +pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + let actionable = actionable_site_replication_retry_events(&runtime.state, OffsetDateTime::now_utc()); + if actionable.is_empty() { + return Ok(()); + } + let Some(store) = current_object_store_handle() else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + // The tick-level gate ran before the reconcilers; a multi-step flow + // (endpoint refresh commits its pending marker without the lifecycle + // guard) may have started since. Re-check on the fresh state. + return Ok(()); + } + // Serialize against operator repair execution. This does NOT close the + // dry-run -> execute window (dry-run takes no lock): a drain settling a + // replayable bucket-op entry in that window changes the preflight token + // and execute fails safe with "preflight is stale" — the operator + // re-runs the dry-run. Lock order matches repair: lifecycle guard (held + // by the reconcile tick) -> repair execution lock -> state object lock + // inside the send bookkeeping. An operator repair holding the lock makes + // this tick skip after the lock-acquire timeout. + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + drain_site_replication_retry_queue_locked(runtime, actionable).await + }) + .await + .map_err(ApiError::from)? +} + +pub(crate) async fn drain_site_replication_retry_queue_locked( + runtime: SiteReplicationRuntime, + events: Vec, +) -> S3Result<()> { + let needs_plan = events + .iter() + .any(|event| !matches!(classify_site_replication_retry_event(event), Some(RetryDrainAction::PeerEdit))); + // The plan is a full local snapshot (buckets + IAM); build it once per + // tick and only when a snapshot resend is actually due. + let plan = if needs_plan { + let info = build_sr_info(&runtime.state, &runtime.local_peer).await?; + Some(site_replication_bootstrap_plan(&info)?) + } else { + None + }; + + let mut events_by_peer: BTreeMap> = BTreeMap::new(); + for event in events { + events_by_peer + .entry(event.peer_deployment_id.clone()) + .or_default() + .push(event); + } + + let mut settled = 0usize; + let mut failures = 0usize; + for (deployment_id, peer_events) in events_by_peer { + let Some(peer) = runtime.state.peers.get(&deployment_id) else { + continue; + }; + if deployment_id == runtime.local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + { + continue; + } + let transport = match PeerTransport::for_runtime_peer(peer).await { + Ok(transport) => transport, + Err(err) => { + // Record the attempt so backoff advances for an unreachable + // peer instead of re-dialing it every tick. + for event in &peer_events { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + } + failures += peer_events.len(); + continue; + } + }; + for event in peer_events { + let Some(action) = classify_site_replication_retry_event(&event) else { + continue; + }; + match drain_one_site_replication_retry_event(&runtime, peer, &transport, &event, action, plan.as_ref()).await { + Ok(true) => settled += 1, + Ok(false) => {} + Err(_) => failures += 1, + } + } + } + + if settled > 0 || failures > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_settled", + settled, + failures, + "admin site replication state" + ); + } + Ok(()) +} + +/// Replay one retry event against its peer. Returns `Ok(true)` when the +/// event was settled (delivered, or provably stale), `Ok(false)` when it was +/// skipped, and `Err` after a failed delivery (already re-queued with an +/// incremented retry count). +pub(crate) async fn drain_one_site_replication_retry_event( + runtime: &SiteReplicationRuntime, + peer: &PeerInfo, + transport: &PeerTransport, + event: &SiteReplicationRetryEvent, + action: RetryDrainAction, + plan: Option<&SiteReplicationBootstrapPlan>, +) -> S3Result { + let access_key = &runtime.state.service_account_access_key; + let secret_key = &runtime.service_account_secret_key; + match action.clone() { + RetryDrainAction::IamSnapshot | RetryDrainAction::BucketMetadataSnapshot => { + let Some(plan) = plan else { + return Ok(false); + }; + let mut current_snapshot = RetrySnapshot::from_plan(&action, plan).expect("snapshot action has a snapshot"); + let mut replay = current_snapshot.clone(); + for _ in 0..SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS { + let current_fingerprint = current_snapshot.fingerprint()?; + if let Err(err) = replay.send(transport, access_key, secret_key).await { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; + let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; + let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); + if fresh_snapshot.fingerprint()? == current_fingerprint { + escalate_site_replication_retry_event_up_to(peer, &event.path, event.updated_at).await; + return Ok(true); + } + replay = RetrySnapshot::replay_after_change(¤t_snapshot, &fresh_snapshot, OffsetDateTime::now_utc()); + current_snapshot = fresh_snapshot; + } + Ok(false) + } + RetryDrainAction::BucketOpReplay { operation, bucket } => { + let Some(plan) = plan else { + return Ok(false); + }; + // Replay from the CURRENT plan, never the recorded path: the + // recorded query can carry an expired one-shot bootstrap token or + // a stale createdAt. + let make_op = operation == SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING; + let paths = if make_op { + &plan.bucket_make_ops + } else { + &plan.bucket_configure_ops + }; + let tasks: Vec> = paths + .iter() + .filter(|path| retry_bucket_name(path).as_deref() == Some(bucket.as_str())) + .map(|path| { + if make_op { + SiteReplicationRepairTask::BucketMake(path) + } else { + SiteReplicationRepairTask::Replication(path) + } + }) + .collect(); + if tasks.is_empty() { + // The bucket left the plan (deleted, or replication no longer + // configured): the recorded intent is stale, settle it. + dequeue_site_replication_retry_event(peer, &event.path).await; + return Ok(true); + } + for task in &tasks { + if let Err(err) = task.send(transport, access_key, secret_key).await { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + } + dequeue_site_replication_retry_event(peer, &event.path).await; + Ok(true) + } + RetryDrainAction::PeerEdit => { + // The recorded generation is stale by definition — the receiver + // fences it. Allocate a fresh generation and re-send the current + // peer records (a superset of the failed body; the receiver + // upserts), all inside one state transaction so the fence and the + // bodies agree. + let target_id = peer.deployment_id.clone(); + let (generation, bodies) = update_site_replication_state(move |state| { + if !state.peers.contains_key(&target_id) { + return Ok((None, Vec::new())); + } + Ok((Some(next_peer_edit_generation(state)), state.peers.values().cloned().collect::>())) + }) + .await?; + let Some(generation) = generation else { + // Peer left between the snapshot and now; the queue entry was + // already pruned by remove_sites. + return Ok(false); + }; + let local_deployment_id = Some(runtime.local_peer.deployment_id.as_str()).filter(|id| !id.is_empty()); + let edit_path = peer_edit_path_with_fence(local_deployment_id, generation); + let delivery_fence = local_deployment_id.is_some().then_some(generation); + for body in &bodies { + if let Err(err) = send_peer_admin_request_with_client( + &transport.client, + &transport.connection, + &edit_path, + access_key, + secret_key, + body, + ) + .await + { + enqueue_site_replication_retry_event_for_generation( + peer, + SITE_REPLICATION_PEER_EDIT_PATH, + &err, + delivery_fence, + ) + .await; + return Err(err); + } + } + dequeue_site_replication_retry_event_for_generation(peer, SITE_REPLICATION_PEER_EDIT_PATH, delivery_fence).await; + Ok(true) + } + } +} + +/// Remove a retry event for (peer, path) from the queue on successful delivery. +/// This is a no-op (load + no-op persist skipped) when no matching entry exists, +/// avoiding unnecessary I/O on the common path. +pub(crate) async fn dequeue_site_replication_retry_event(peer: &PeerInfo, path: &str) { + dequeue_site_replication_retry_event_for_generation(peer, path, None).await +} + +pub(crate) async fn dequeue_site_replication_retry_event_for_generation(peer: &PeerInfo, path: &str, generation: Option) { + let result = async { + // Fast path: this sits on every successful hook broadcast, so probe + // with a plain read first and only enter the locked RMW on a hit + // (the transaction re-checks under the lock). + let mut probe = load_site_replication_state().await?; + if settle_site_replication_retry_events(&mut probe.retry_queue, peer, path, generation) == 0 { + return Ok(()); + } + let peer_owned = peer.clone(); + let path_owned = path.to_string(); + update_site_replication_state(move |state| { + settle_site_replication_retry_events(&mut state.retry_queue, &peer_owned, &path_owned, generation); + Ok(()) + }) + .await?; + Ok::<_, S3Error>(()) + } + .await; + + if let Err(err) = result { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + peer = %peer.endpoint, + deployment_id = %peer.deployment_id, + path, + error = ?err, + "failed to dequeue site replication retry event" + ); + } +} diff --git a/rustfs/src/site_replication/state.rs b/rustfs/src/site_replication/state.rs new file mode 100644 index 000000000..cb5881fa4 --- /dev/null +++ b/rustfs/src/site_replication/state.rs @@ -0,0 +1,593 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub(crate) const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit"; + +/// Peer-edit fencing token, carried as query parameters so a peer that predates +/// the fence simply ignores them (unknown query keys are dropped) and keeps the +/// previous last-writer-wins behaviour. +pub(crate) const SITE_REPLICATION_EDIT_ORIGIN_QUERY: &str = "editOrigin"; + +pub(crate) const SITE_REPLICATION_EDIT_GENERATION_QUERY: &str = "editGeneration"; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct SiteReplicationState { + pub(crate) name: String, + pub(crate) service_account_access_key: String, + #[serde(default, skip_serializing)] + pub(crate) service_account_secret_key: String, + pub(crate) service_account_parent: String, + pub(crate) peers: BTreeMap, + pub(crate) updated_at: Option, + pub(crate) resync_status: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) pending_rotation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) pending_remove: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) pending_endpoint_refresh: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) retry_queue: Vec, + #[serde(default)] + pub(crate) sync_state_initialized: bool, + /// Fencing token for peer-edit delivery, allocated inside the state + /// transaction (the distributed state-object lock). Two nodes of THIS + /// site that accept admin edits concurrently therefore get strictly + /// ordered generations, and a delivery that stalls can be recognised as + /// stale by the receiving site. + #[serde(default)] + pub(crate) edit_generation: u64, + /// Per-origin high-water mark of the peer edits already applied here, + /// keyed by the origin site's deployment id. A delivery whose generation + /// is not above the mark arrived out of order and must not overwrite the + /// newer edit that already landed. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) applied_edit_generations: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct PendingEndpointRefresh { + pub(crate) id: String, + pub(crate) peer: PeerInfo, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) remote_peers: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub(crate) acked_deployment_ids: BTreeSet, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct PendingRotation { + pub(crate) id: String, + pub(crate) access_key: String, + pub(crate) parent: String, + pub(crate) new_secret_key: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) secret_candidates: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) peers: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub(crate) acked_deployment_ids: BTreeSet, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) updated_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct PendingRemove { + pub(crate) id: String, + pub(crate) req: SRRemoveReq, + pub(crate) service_account_access_key: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) secret_candidates: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) original_peers: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub(crate) acked_deployment_ids: BTreeSet, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub(crate) updated_at: Option, +} + +impl SiteReplicationState { + pub(crate) fn enabled(&self) -> bool { + self.peers.len() > 1 + } +} + +pub(crate) fn parse_site_replication_state(data: &[u8]) -> S3Result { + let mut state: SiteReplicationState = serde_json::from_slice(data) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}")))?; + state.peers = normalize_peer_map_by_identity(state.peers); + // A peer-edit high-water mark only fences a CURRENT peer. A site that + // leaves drops below two peers, which clears its own state object and + // restarts its generation counter — a mark left over from the previous + // membership must not reject the edits it sends after it rejoins. This + // pruning covers departures THIS site observed; an origin removed + // unilaterally elsewhere stays in this peer map with its mark, and the + // wall-clock floor in `next_peer_edit_generation` is what lifts its + // restarted counter over that mark. Dropping departed origins on load + // also keeps the map bounded. + state + .applied_edit_generations + .retain(|origin, _| state.peers.contains_key(origin)); + if !state.sync_state_initialized { + if state.enabled() { + mark_unknown_peer_sync_enabled(&mut state.peers); + } + state.sync_state_initialized = true; + } + Ok(state) +} + +pub(crate) async fn load_site_replication_state() -> S3Result { + let Some(store) = current_object_store_handle() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + match read_admin_config(store, SITE_REPLICATION_STATE_PATH).await { + Ok(data) => parse_site_replication_state(&data), + Err(StorageError::ConfigNotFound) => Ok(SiteReplicationState::default()), + Err(err) => Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load site replication state: {err}"), + )), + } +} + +/// Whether this deployment participates in site replication (two or more +/// peers in the persisted state). Read by the S3 interface layer to gate +/// replication-config edits (MinIO `ErrReplicationDenyEditError` semantics, +/// issue #1948); a state-read failure propagates so the gate fails closed. +pub(crate) async fn site_replication_enabled() -> S3Result { + Ok(load_site_replication_state().await?.enabled()) +} + +/// Deployment ids of the remote peers the reconciler derives a +/// `site-repl-` rule for on every bucket (the same peer filter as +/// `build_site_replication_config`); empty when site replication is not +/// enabled. Read by the bucket usecase so an S3 replication-config edit keeps +/// exactly the reconciler-owned rules (issue #1948); a state-read failure +/// propagates so the edit fails closed. +pub(crate) async fn site_replication_edit_context() -> S3Result<(HashSet, OperatorRuleContract)> { + let Some(runtime) = runtime_site_replication_targets().await? else { + // Enabled without a service account is a state this site cannot + // broadcast from either; the peers are still the reconciler's. + let state = load_site_replication_state().await?; + if !state.enabled() { + return Ok((HashSet::new(), OperatorRuleContract::Derived)); + } + let peers = remote_peer_deployment_ids(&state, ¤t_local_runtime_peer(&state)); + return Ok((peers, OperatorRuleContract::Legacy)); + }; + let peers = remote_peer_deployment_ids(&runtime.state, &runtime.local_peer); + let contract = site_replication_operator_rule_contract(&runtime).await; + Ok((peers, contract)) +} + +/// Whether every remote peer merges replication configs under the derived +/// contract, probed through the peer capability endpoint. A peer that does +/// not (or cannot be asked) pins the cluster to [`OperatorRuleContract::Legacy`] +/// for this edit: consistency across sites wins over keeping the operator's +/// priority values, and the legacy merge keeps their order anyway. +pub(crate) async fn site_replication_operator_rule_contract(runtime: &SiteReplicationRuntime) -> OperatorRuleContract { + let remote_peers: Vec<&PeerInfo> = runtime + .state + .peers + .values() + .filter(|peer| { + peer.deployment_id != runtime.local_peer.deployment_id + && !same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + }) + .collect(); + let probes = futures::future::join_all(remote_peers.iter().map(|peer| async move { + let transport = PeerTransport::for_runtime_peer(peer).await?; + let (status, body) = send_peer_admin_request_raw_with_client( + &transport.client, + &transport.connection, + SITE_REPLICATION_PEER_DERIVED_RULE_CONTRACT_CAPABILITY_PATH, + &runtime.state.service_account_access_key, + &runtime.service_account_secret_key, + &(), + ) + .await?; + peer_capability_response_supported(peer, status, &body) + })) + .await; + operator_rule_contract_from_probes(remote_peers.into_iter().zip(probes)) +} + +pub(crate) fn operator_rule_contract_from_probes<'a>( + probes: impl IntoIterator)>, +) -> OperatorRuleContract { + for (peer, probe) in probes { + match probe { + Ok(true) => {} + Ok(false) => return OperatorRuleContract::Legacy, + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "derived_rule_contract_probe_failed", + peer = %peer.endpoint, + error = %err, + "admin site replication state" + ); + return OperatorRuleContract::Legacy; + } + } + } + OperatorRuleContract::Derived +} + +pub(crate) fn remote_peer_deployment_ids(state: &SiteReplicationState, local_peer: &PeerInfo) -> HashSet { + state + .peers + .values() + .filter(|peer| { + peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + }) + .map(|peer| peer.deployment_id.clone()) + .collect() +} + +/// Deployment ids of every site in the cluster, this one included: the set +/// a peer's derived rules can name (its rule towards this site carries this +/// site's id). Empty when site replication is not enabled. +pub(crate) async fn site_replication_deployment_ids() -> S3Result> { + let state = load_site_replication_state().await?; + if !state.enabled() { + return Ok(HashSet::new()); + } + Ok(state.peers.values().map(|peer| peer.deployment_id.clone()).collect()) +} + +pub(crate) async fn load_site_replication_state_no_lock(store: Arc) -> S3Result { + match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await { + Ok(data) => parse_site_replication_state(&data), + Err(StorageError::ConfigNotFound) => Ok(SiteReplicationState::default()), + Err(err) => Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load site replication state: {err}"), + )), + } +} + +/// Persist-or-clear under an already-held state object lock. Normalizes the +/// peer map exactly once (the historical persist path normalized twice with +/// two full clones — P2-22). +pub(crate) async fn persist_site_replication_state_no_lock(store: Arc, mut state: SiteReplicationState) -> S3Result<()> { + state.peers = normalize_peer_map_by_identity(state.peers); + if state.peers.len() <= 1 && state.pending_rotation.is_none() && state.pending_remove.is_none() { + match delete_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await { + Ok(()) | Err(StorageError::ConfigNotFound) => Ok(()), + Err(err) => Err(S3Error::with_message(S3ErrorCode::InternalError, format!("clear state failed: {err}"))), + } + } else { + let data = serde_json::to_vec(&state) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize state failed: {e}")))?; + save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, data) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save state failed: {e}"))) + } +} + +/// What a state transaction closure decided to do with the state it was +/// handed. `Unchanged` skips the write entirely: the ack markers and the +/// pending-clearing paths run on every retry and mostly find their pending id +/// already gone, and the retry queue shares this object — rewriting it byte +/// for byte only makes those misses contend with the writers that do have +/// something to say. +pub(crate) enum StateCommit { + Changed(T), + Unchanged(T), +} + +/// The site-replication state RMW transaction: load, mutate, persist — all +/// under the distributed state-object write lock (see +/// crate::site_replication::state_lock). No peer network calls and no other +/// config locks inside `update`; anything that has to talk to a peer belongs +/// between two transactions, with the precondition re-checked inside the +/// second one. +pub(crate) async fn update_site_replication_state(update: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(&mut SiteReplicationState) -> S3Result + Send + 'static, +{ + update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await +} + +/// [`update_site_replication_state`] for closures that may find nothing to +/// do — see [`StateCommit`]. +pub(crate) async fn update_site_replication_state_when_changed(update: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(&mut SiteReplicationState) -> S3Result> + Send + 'static, +{ + with_site_replication_state_lock(move || async move { + let store = current_object_store_handle() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let mut state = load_site_replication_state_no_lock(store.clone()).await?; + match update(&mut state)? { + StateCommit::Changed(result) => { + persist_site_replication_state_no_lock(store, state).await?; + Ok(result) + } + StateCommit::Unchanged(result) => Ok(result), + } + }) + .await +} + +/// Test-only seeding of the state object. Every production write goes through +/// [`update_site_replication_state`] — this helper is `cfg(test)` so a new +/// call site cannot reintroduce the pre-P1-15 shape (load through one object +/// lock, save through another, with the mutation in between unprotected). +#[cfg(test)] +pub(crate) async fn save_site_replication_state(state: &SiteReplicationState) -> S3Result<()> { + let Some(store) = current_object_store_handle() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let mut normalized = state.clone(); + normalized.peers = normalize_peer_map_by_identity(normalized.peers); + + let data = serde_json::to_vec(&normalized) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize state failed: {e}")))?; + save_admin_config(store, SITE_REPLICATION_STATE_PATH, data) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save state failed: {e}")))?; + Ok(()) +} + +pub(crate) fn request_endpoint(uri: &Uri, headers: &HeaderMap) -> String { + let scheme = get_source_scheme(headers) + .and_then(|value| { + value + .split(',') + .next() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_ascii_lowercase) + }) + .or_else(|| uri.scheme_str().map(str::to_ascii_lowercase)) + .unwrap_or_else(|| { + if runtime_tls_enabled() { + "https".to_string() + } else { + "http".to_string() + } + }); + + let host = headers + .get(http::header::HOST) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| uri.authority().map(|value| value.as_str().to_string())) + .or_else(|| { + current_endpoints_handle().and_then(|endpoints| { + endpoints + .as_ref() + .iter() + .flat_map(|pool| pool.endpoints.as_ref().iter()) + .find(|endpoint| endpoint.is_local) + .map(|endpoint| endpoint.host_port()) + }) + }) + .unwrap_or_else(|| format!("127.0.0.1:{}", current_runtime_port())); + + format!("{scheme}://{host}") +} + +pub(crate) fn runtime_console_port() -> Option { + let console_address = get_config_snapshot() + .map(|snapshot| snapshot.console_address.clone()) + .unwrap_or_else(|| rustfs_utils::get_env_str(ENV_RUSTFS_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ADDRESS)); + + let parse_target = if console_address.starts_with(':') { + format!("127.0.0.1{console_address}") + } else { + console_address + }; + + Url::parse(&format!("http://{parse_target}")) + .ok() + .and_then(|parsed| parsed.port_or_known_default()) +} + +pub(crate) fn site_replication_local_endpoint(uri: &Uri, headers: &HeaderMap) -> String { + let endpoint = request_endpoint(uri, headers); + match Url::parse(&endpoint) { + Ok(mut parsed) => { + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return request_endpoint(&Uri::from_static("/"), &HeaderMap::new()); + } + if parsed.port_or_known_default() == runtime_console_port() && parsed.set_port(Some(current_runtime_port())).is_ok() { + parsed.to_string().trim_end_matches('/').to_string() + } else { + endpoint + } + } + Err(_) => request_endpoint(&Uri::from_static("/"), &HeaderMap::new()), + } +} + +pub(crate) fn current_local_runtime_endpoint() -> String { + site_replication_local_endpoint(&Uri::from_static("/"), &HeaderMap::new()) +} + +pub(crate) fn infer_site_name(endpoint: &str) -> String { + endpoint + .trim_start_matches("http://") + .trim_start_matches("https://") + .split('/') + .next() + .unwrap_or_default() + .split(':') + .next() + .unwrap_or_default() + .to_string() +} + +pub(crate) fn stored_peer_tls_settings(stored_peer: Option<&PeerInfo>) -> (bool, String) { + stored_peer + .map(|peer| (peer.skip_tls_verify, peer.ca_cert_pem.clone())) + .unwrap_or_default() +} + +/// The local peer record as the given state describes it. Split out of +/// [`current_local_peer`] so a state transaction can rebuild it against the +/// state it just loaded: the request the endpoint came from cannot cross into +/// the transaction closure, but the endpoint itself can. +pub(crate) fn local_peer_at_endpoint(endpoint: String, state: &SiteReplicationState) -> PeerInfo { + let deployment_id = current_deployment_id().unwrap_or_else(|| deployment_id_for_endpoint(&endpoint)); + let stored_peer = state.peers.get(&deployment_id); + let (skip_tls_verify, ca_cert_pem) = stored_peer_tls_settings(stored_peer); + + PeerInfo { + endpoint: endpoint.clone(), + name: if state.name.is_empty() { + stored_peer + .map(|peer| peer.name.clone()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| infer_site_name(&endpoint)) + } else { + state.name.clone() + }, + deployment_id, + sync_state: stored_peer.map(|peer| peer.sync_state.clone()).unwrap_or(SyncStatus::Unknown), + default_bandwidth: stored_peer.map(|peer| peer.default_bandwidth.clone()).unwrap_or_default(), + replicate_ilm_expiry: stored_peer.is_some_and(|peer| peer.replicate_ilm_expiry), + object_naming_mode: stored_peer.map(|peer| peer.object_naming_mode.clone()).unwrap_or_default(), + skip_tls_verify, + ca_cert_pem, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + } +} + +pub(crate) fn current_local_runtime_peer(state: &SiteReplicationState) -> PeerInfo { + local_peer_at_endpoint(current_local_runtime_endpoint(), state) +} + +pub(crate) fn normalize_peer_map_by_identity(peers: BTreeMap) -> BTreeMap { + normalize_peer_map_by_identity_with(peers, normalize_peer_info) +} + +pub(crate) fn normalize_peer_info(mut peer: PeerInfo) -> PeerInfo { + if peer.deployment_id.is_empty() { + peer.deployment_id = deployment_id_for_endpoint(&peer.endpoint); + } + if peer.name.is_empty() { + peer.name = infer_site_name(&peer.endpoint); + } + if peer.api_version.is_none() { + peer.api_version = Some(SITE_REPL_API_VERSION.to_string()); + } + peer +} + +pub(crate) async fn site_replicator_service_account_secret(access_key: &str) -> S3Result { + let Some(iam_sys) = current_iam_handle() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + iam_sys + .get_site_replicator_service_account_secret(access_key) + .await + .map_err(ApiError::from) + .map_err(Into::into) +} + +pub(crate) fn legacy_site_replicator_state_secret(state: &SiteReplicationState) -> Option { + (state.service_account_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT && !state.service_account_secret_key.is_empty()) + .then(|| state.service_account_secret_key.clone()) +} + +pub(crate) fn pending_endpoint_refresh(state: &SiteReplicationState) -> Option { + state.pending_endpoint_refresh.clone().or_else(|| { + state + .retry_queue + .iter() + .find(|event| event.path == SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH) + .and_then(|event| serde_json::from_str(&event.last_error).ok()) + }) +} + +/// The wall clock in unix nanoseconds, clamped into u64. A pre-1970 (or +/// post-2554) clock yields 0, which makes the hybrid allocation below +/// degrade to the plain `previous + 1` counter — monotone, never panicking. +pub(crate) fn edit_generation_wall_clock() -> u64 { + u64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(0) +} + +/// Allocate the next peer-edit generation as a hybrid logical clock: +/// `max(wall clock in unix nanoseconds, previous + 1)`. Called inside the +/// state transaction, so the value is handed out under the distributed +/// state-object lock and two nodes of this site can never take the same one +/// (`previous + 1` keeps the sequence strictly increasing even when two +/// allocations land in one clock tick, and keeps it monotone on a node +/// whose clock stepped backwards mid-lifetime). +/// +/// The wall-clock floor is what survives the counter's death. A site +/// removed while unreachable — the receiver never dropped it from its peer +/// map, so the load-time mark pruning in `parse_site_replication_state` +/// never fired — that later rejoins recreates its state object with the +/// counter back at zero. A plain counter would then hand out generations +/// below the receiver's stale high-water mark and every delivery would be +/// silently fenced until the counter caught up. Jumping to wall time clears +/// that mark: every value the deleted lifetime handed out was capped by the +/// wall clock at its own allocation (or by a prior lifetime's cap, applied +/// inductively), so the recreated lifetime's first allocation exceeds them +/// all — while a pre-removal delivery still in flight stays below the new +/// floor and remains correctly fenced. Marks recorded by pre-hybrid +/// receivers (small plain-counter values) sit far below any wall-clock +/// value, so a restarted origin passes those too — the fix needs only the +/// sender upgraded, nothing on the wire or in the receiver changed. +/// +/// A wall clock that regresses across a delete/recreate (the recreating +/// node's clock behind the clock that fed the previous lifetime) mints +/// below the stale mark and the origin stays fenced — but only until real +/// time passes the previous lifetime's last allocation, because every later +/// allocation takes the wall-clock floor again (and never longer than +/// [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`]: a regression past the window +/// leaves the mark implausibly distant and the origin runs unfenced +/// immediately). Bounded by the skew, +/// self-healing, and no rollback window beyond the plain counter's: a +/// delivery applies only at or above the receiver's mark, so the one +/// cross-lifetime interleaving that can apply stale content — a +/// pre-removal delivery whose generation lands above everything the +/// regressed new lifetime has minted — required the same straggler landing +/// above the mark under the plain counter, where the recreated counter's +/// low restart made it strictly easier to hit. +pub(crate) fn next_peer_edit_generation(state: &mut SiteReplicationState) -> u64 { + state.edit_generation = edit_generation_wall_clock().max(state.edit_generation.saturating_add(1)); + state.edit_generation +} + +/// Build the peer-edit request path carrying the fencing token. The bare +/// constant stays the retry-queue key: the query only fences the wire +/// delivery, and a per-generation key would make every retry event unique. +/// Without a local deployment id there is nothing to fence against, so the +/// unstamped path is sent and the receiver keeps its pre-fence behaviour. +pub(crate) fn peer_edit_path_with_fence(origin: Option<&str>, generation: u64) -> String { + let Some(origin) = origin.filter(|origin| !origin.is_empty()) else { + return SITE_REPLICATION_PEER_EDIT_PATH.to_string(); + }; + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair(SITE_REPLICATION_EDIT_ORIGIN_QUERY, origin) + .append_pair(SITE_REPLICATION_EDIT_GENERATION_QUERY, &generation.to_string()) + .finish(); + format!("{SITE_REPLICATION_PEER_EDIT_PATH}?{query}") +} diff --git a/rustfs/src/admin/site_replication_state.rs b/rustfs/src/site_replication/state_lock.rs similarity index 93% rename from rustfs/src/admin/site_replication_state.rs rename to rustfs/src/site_replication/state_lock.rs index d39618a2b..6f9cc7bfd 100644 --- a/rustfs/src/admin/site_replication_state.rs +++ b/rustfs/src/site_replication/state_lock.rs @@ -34,12 +34,11 @@ //! Lock order: lifecycle -> bucket operation -> repair admission //! -> state object lock -> per-bucket metadata. -use crate::admin::storage_api::runtime::ECStore; -use crate::admin::storage_api::s3::{S3Error, S3ErrorCode, S3Result}; -use crate::storage::storage_api::with_config_object_write_lock; +use super::{S3Error, S3ErrorCode, S3Result}; +use crate::storage_api::site_replication::{ECStore, with_config_object_write_lock}; use std::sync::Arc; -use super::runtime_sources::current_object_store_handle; +use crate::runtime_sources::current_object_store_handle; /// Config object holding the whole site-replication state, including the /// retry-event queue. Shared by the typed handler-side accessors and the diff --git a/rustfs/src/site_replication/transport.rs b/rustfs/src/site_replication/transport.rs new file mode 100644 index 000000000..417e75762 --- /dev/null +++ b/rustfs/src/site_replication/transport.rs @@ -0,0 +1,1010 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub(crate) const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +pub(crate) const SITE_REPLICATION_PEER_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); + +pub(crate) const SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT: usize = 256; + +pub(crate) const MAX_PEER_CA_CERT_PEM_SIZE: usize = 256 * 1024; + +pub(crate) const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET"; + +pub(crate) const SITE_REPLICATION_PEER_DERIVED_RULE_CONTRACT_CAPABILITY_PATH: &str = + "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=derived-rule-contract"; + +pub(crate) const RUSTFS_ADMIN_V3_PREFIX: &str = "/rustfs/admin/v3"; + +pub(crate) const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3"; + +pub(crate) const MINIO_SITE_REPLICATION_PEER_JOIN_PATH: &str = "/minio/admin/v3/site-replication/peer/join"; + +#[derive(Clone)] +pub(crate) enum SiteReplicationPeerClientCacheEntry { + Ready(reqwest::Client), + Failed(String), +} + +#[derive(Clone)] +pub(crate) struct SiteReplicationPeerClientCache { + pub(crate) generation: u64, + pub(crate) entry: SiteReplicationPeerClientCacheEntry, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PeerConnection { + pub(crate) endpoint: Url, + pub(crate) skip_tls_verify: bool, + pub(crate) ca_cert_pem: String, +} + +#[derive(Deserialize, Default)] +pub(crate) struct PeerTlsFieldPresence { + #[serde(rename = "skipTlsVerify")] + pub(crate) skip_tls_verify: Option, + #[serde(rename = "caCertPem")] + pub(crate) ca_cert_pem: Option, +} + +impl PeerTlsFieldPresence { + pub(crate) fn has_skip_tls_verify(&self) -> bool { + self.skip_tls_verify.is_some() + } + + pub(crate) fn has_ca_cert_pem(&self) -> bool { + self.ca_cert_pem.is_some() + } +} + +#[derive(Clone)] +pub(crate) struct PeerDnsResolver { + pub(crate) allow_loopback: bool, + #[cfg(test)] + pub(crate) overrides: Option>>>, +} + +impl PeerDnsResolver { + pub(crate) fn new(allow_loopback: bool) -> Self { + Self { + allow_loopback, + #[cfg(test)] + overrides: None, + } + } + + #[cfg(test)] + pub(crate) fn with_overrides(allow_loopback: bool, overrides: HashMap>) -> Self { + Self { + allow_loopback, + overrides: Some(Arc::new(overrides)), + } + } +} + +impl reqwest::dns::Resolve for PeerDnsResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let host = name.as_str().to_string(); + let allow_loopback = self.allow_loopback; + #[cfg(test)] + let overrides = self.overrides.clone(); + Box::pin(async move { + #[cfg(test)] + let overridden = overrides.as_ref().and_then(|entries| entries.get(&host)).cloned(); + #[cfg(not(test))] + let overridden: Option> = None; + + let ips = if let Some(ips) = overridden { + ips + } else { + tokio::net::lookup_host((host.as_str(), 0)) + .await? + .map(|addr| addr.ip()) + .collect() + }; + let addrs = ips + .into_iter() + .filter(|ip| resolved_peer_ip_allowed(&host, *ip, allow_loopback)) + .map(|ip| SocketAddr::new(ip, 0)) + .collect::>(); + if addrs.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!("site replication DNS resolution for `{host}` returned no allowed addresses"), + ) + .into()); + } + Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +impl PeerConnection { + pub(crate) fn new(endpoint: &str, skip_tls_verify: bool, ca_cert_pem: &str) -> S3Result { + validate_peer_connection_inner(endpoint, skip_tls_verify, ca_cert_pem, loopback_replication_targets_allowed()) + } + + pub(crate) fn endpoint(&self) -> &str { + self.endpoint.as_str().trim_end_matches('/') + } + + pub(crate) fn uses_default_tls(&self) -> bool { + !self.skip_tls_verify && self.ca_cert_pem.is_empty() + } +} + +impl TryFrom<&PeerInfo> for PeerConnection { + type Error = S3Error; + + fn try_from(peer: &PeerInfo) -> Result { + Self::new(&peer.endpoint, peer.skip_tls_verify, &peer.ca_cert_pem) + } +} + +impl TryFrom<&PeerSite> for PeerConnection { + type Error = S3Error; + + fn try_from(site: &PeerSite) -> Result { + Self::new(&site.endpoint, site.skip_tls_verify, &site.ca_cert_pem) + } +} + +static SITE_REPLICATION_PEER_CLIENT: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + +pub(crate) fn site_replication_peer_client_cache_hit( + cache: &Option, + generation: u64, +) -> Option> { + let cached = cache.as_ref()?; + if cached.generation != generation { + return None; + } + Some(match &cached.entry { + SiteReplicationPeerClientCacheEntry::Ready(client) => Ok(client.clone()), + SiteReplicationPeerClientCacheEntry::Failed(err) => Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("initialize site replication peer client failed: {err}"), + )), + }) +} + +pub(crate) struct SiteReplicationRuntime { + pub(crate) state: SiteReplicationState, + pub(crate) local_peer: PeerInfo, + pub(crate) service_account_secret_key: String, +} + +pub(crate) fn build_site_replication_peer_client(outbound_tls: &GlobalPublishedOutboundTlsState) -> S3Result { + build_site_replication_peer_client_with_resolver(outbound_tls, PeerDnsResolver::new(loopback_replication_targets_allowed())) +} + +pub(crate) fn build_site_replication_peer_client_with_resolver( + outbound_tls: &GlobalPublishedOutboundTlsState, + resolver: PeerDnsResolver, +) -> S3Result { + let mut builder = reqwest::Client::builder() + .no_proxy() + .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT) + .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT) + .pool_idle_timeout(Some(Duration::from_secs(60))) + .redirect(reqwest::redirect::Policy::none()) + .dns_resolver(resolver); + + if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { + let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); + let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to parse published site-replication CA certs: {e}"), + ) + })?; + + for cert_der in certs_der { + let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load published site-replication CA cert: {e}"), + ) + })?; + builder = builder.add_root_certificate(cert); + } + } + + builder + .build() + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}"))) +} + +pub(crate) fn build_custom_site_replication_peer_client( + outbound_tls: &GlobalPublishedOutboundTlsState, + connection: &PeerConnection, +) -> S3Result { + build_custom_site_replication_peer_client_with_resolver( + outbound_tls, + connection, + PeerDnsResolver::new(loopback_replication_targets_allowed()), + ) +} + +pub(crate) fn build_custom_site_replication_peer_client_with_resolver( + outbound_tls: &GlobalPublishedOutboundTlsState, + connection: &PeerConnection, + resolver: PeerDnsResolver, +) -> S3Result { + let mut builder = reqwest::Client::builder() + .no_proxy() + .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT) + .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT) + .pool_idle_timeout(Some(Duration::from_secs(60))) + .redirect(reqwest::redirect::Policy::none()) + .dns_resolver(resolver) + .danger_accept_invalid_certs(connection.skip_tls_verify); + + if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { + let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); + let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to parse published site-replication CA certs: {e}"), + ) + })?; + for cert_der in certs_der { + let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load published site-replication CA cert: {e}"), + ) + })?; + builder = builder.add_root_certificate(cert); + } + } + if !connection.ca_cert_pem.is_empty() { + for cert in parse_peer_ca_certificates(&connection.ca_cert_pem)? { + builder = builder.add_root_certificate(cert); + } + } + + builder + .build() + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}"))) +} + +pub(crate) async fn site_replication_peer_client() -> S3Result { + let generation = current_outbound_tls_generation().0; + let cache = SITE_REPLICATION_PEER_CLIENT.lock().await; + if let Some(hit) = site_replication_peer_client_cache_hit(&cache, generation) { + return hit; + } + drop(cache); + + let outbound_tls = current_outbound_tls_state().await; + let built = build_site_replication_peer_client(&outbound_tls); + let cache_entry = match &built { + Ok(client) => SiteReplicationPeerClientCacheEntry::Ready(client.clone()), + Err(err) => SiteReplicationPeerClientCacheEntry::Failed(err.to_string()), + }; + + let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; + if cache.as_ref().is_none_or(|cached| cached.generation <= generation) { + *cache = Some(SiteReplicationPeerClientCache { + generation, + entry: cache_entry, + }); + } + + built +} + +pub(crate) async fn site_replication_client_for(connection: &PeerConnection) -> S3Result { + // Revalidate at the client boundary so callers cannot bypass endpoint/TLS policy. + let connection = PeerConnection::new(connection.endpoint(), connection.skip_tls_verify, &connection.ca_cert_pem)?; + if connection.uses_default_tls() { + return site_replication_peer_client().await; + } + let outbound_tls = current_outbound_tls_state().await; + build_custom_site_replication_peer_client(&outbound_tls, &connection) +} + +pub(crate) fn runtime_peer_connection(peer: &PeerInfo) -> S3Result { + PeerConnection::try_from(peer).map_err(|err| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("invalid persisted site replication peer `{}`: {err}", peer.endpoint), + ) + }) +} + +pub(crate) struct PeerTransport { + pub(crate) connection: PeerConnection, + pub(crate) client: reqwest::Client, +} + +impl PeerTransport { + pub(crate) async fn for_runtime_peer(peer: &PeerInfo) -> S3Result { + let connection = runtime_peer_connection(peer)?; + let client = site_replication_client_for(&connection).await.map_err(|err| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("initialize persisted site replication peer `{}` transport failed: {err}", peer.endpoint), + ) + })?; + Ok(Self { connection, client }) + } +} + +pub(crate) fn runtime_tls_enabled_with(endpoints: Option<&crate::storage_api::site_replication::EndpointServerPools>) -> bool { + if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() { + return true; + } + + if let Some(tls_enabled) = endpoints.and_then(|endpoints| { + endpoints + .as_ref() + .iter() + .flat_map(|pool| pool.endpoints.as_ref().iter()) + .find(|endpoint| endpoint.is_local) + .map(|endpoint| endpoint.url.scheme().eq_ignore_ascii_case("https")) + }) { + return tls_enabled; + } + + false +} + +pub(crate) fn runtime_tls_enabled() -> bool { + let endpoints = current_endpoints_handle(); + runtime_tls_enabled_with(endpoints.as_ref()) +} + +pub(crate) fn hash_client_secret(secret: Option<&str>) -> String { + let Some(secret) = secret.filter(|secret| !secret.is_empty()) else { + return String::new(); + }; + + let mut hasher = Sha256::new(); + hasher.update(secret.as_bytes()); + URL_SAFE_NO_PAD.encode_to_string(hasher.finalize()) +} + +pub(crate) fn loopback_replication_targets_allowed() -> bool { + std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV) + .map(|value| value.eq_ignore_ascii_case("true") || value == "1") + .unwrap_or(false) +} + +pub(crate) fn validate_peer_egress(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> { + match validate_outbound_url(url) { + Ok(()) => Ok(()), + Err(OutboundUrlError::ForbiddenHost { + reason: "private address", + .. + }) => Ok(()), + Err(OutboundUrlError::ForbiddenHost { + reason: "loopback address" | "loopback host", + .. + }) if allow_loopback && peer_url_has_canonical_loopback_host(url) => Ok(()), + Err(err) => Err(err), + } +} + +pub(crate) fn peer_url_has_canonical_loopback_host(url: &Url) -> bool { + match url.host() { + Some(url::Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), + Some(url::Host::Ipv4(ip)) => ip == std::net::Ipv4Addr::LOCALHOST, + Some(url::Host::Ipv6(ip)) => ip == std::net::Ipv6Addr::LOCALHOST, + None => false, + } +} + +pub(crate) fn resolved_peer_ip_allowed(host: &str, ip: IpAddr, allow_loopback: bool) -> bool { + let Ok(ip_url) = (match ip { + IpAddr::V4(ip) => Url::parse(&format!("http://{ip}")), + IpAddr::V6(ip) => Url::parse(&format!("http://[{ip}]")), + }) else { + return false; + }; + match validate_outbound_url(&ip_url) { + Ok(()) => true, + Err(OutboundUrlError::ForbiddenHost { + reason: "private address", + .. + }) => true, + Err(OutboundUrlError::ForbiddenHost { + reason: "loopback address", + .. + }) => { + allow_loopback + && host.eq_ignore_ascii_case("localhost") + && matches!(ip, IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) | IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)) + } + Err(_) => false, + } +} + +pub(crate) fn parse_peer_ca_certificates(ca_cert_pem: &str) -> S3Result> { + if ca_cert_pem.len() > MAX_PEER_CA_CERT_PEM_SIZE { + return Err(s3_error!(InvalidRequest, "site replication CA certificate exceeds 256 KiB")); + } + if ca_cert_pem.contains("PRIVATE KEY-----") { + return Err(s3_error!( + InvalidRequest, + "site replication CA certificate must not contain a private key" + )); + } + + let mut reader = std::io::BufReader::new(ca_cert_pem.as_bytes()); + let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication CA certificate: {e}")) + })?; + if certs_der.is_empty() { + return Err(s3_error!( + InvalidRequest, + "site replication CA certificate must contain at least one certificate" + )); + } + + let mut root_store = rustls::RootCertStore::empty(); + certs_der + .into_iter() + .map(|cert| { + root_store.add(cert.clone()).map_err(|e| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication CA certificate: {e}")) + })?; + reqwest::Certificate::from_der(cert.as_ref()).map_err(|e| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication CA certificate: {e}")) + }) + }) + .collect() +} + +pub(crate) fn validate_peer_connection_inner( + endpoint: &str, + skip_tls_verify: bool, + ca_cert_pem: &str, + allow_loopback: bool, +) -> S3Result { + let parsed = Url::parse(endpoint) + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site endpoint `{endpoint}`: {e}")))?; + match parsed.scheme() { + "http" | "https" => {} + scheme => { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("invalid site endpoint `{endpoint}`: unsupported scheme `{scheme}`"), + )); + } + } + if parsed.host_str().is_none() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("invalid site endpoint `{endpoint}`: missing host"), + )); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err(s3_error!(InvalidRequest, "invalid site endpoint `{endpoint}`: userinfo is not allowed")); + } + if parsed.path() != "/" || parsed.query().is_some() || parsed.fragment().is_some() { + return Err(s3_error!( + InvalidRequest, + "invalid site endpoint `{endpoint}`: endpoint must be an origin" + )); + } + validate_peer_egress(&parsed, allow_loopback) + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site endpoint `{endpoint}`: {e}")))?; + + if ca_cert_pem.len() > MAX_PEER_CA_CERT_PEM_SIZE { + return Err(s3_error!(InvalidRequest, "site replication CA certificate exceeds 256 KiB")); + } + let ca_cert_pem = ca_cert_pem.trim(); + if parsed.scheme() != "https" && (skip_tls_verify || !ca_cert_pem.is_empty()) { + return Err(s3_error!(InvalidRequest, "site replication TLS settings require an HTTPS endpoint")); + } + if skip_tls_verify && !ca_cert_pem.is_empty() { + return Err(s3_error!(InvalidRequest, "skipTLSVerify and caCertPem are mutually exclusive")); + } + if !ca_cert_pem.is_empty() { + parse_peer_ca_certificates(ca_cert_pem)?; + } + + Ok(PeerConnection { + endpoint: parsed, + skip_tls_verify, + ca_cert_pem: ca_cert_pem.to_string(), + }) +} + +pub(crate) fn site_replication_peer_wire_path(path: &str) -> String { + let (path_only, query) = path + .split_once('?') + .map(|(path, query)| (path, Some(query))) + .unwrap_or((path, None)); + let wire_path = if let Some(suffix) = path_only.strip_prefix(RUSTFS_ADMIN_V3_PREFIX) { + format!("{MINIO_ADMIN_V3_PREFIX}{suffix}") + } else { + path_only.to_string() + }; + + match query { + Some(query) => format!("{wire_path}?{query}"), + None => wire_path, + } +} + +pub(crate) fn site_replication_peer_payload_encrypted(wire_path: &str) -> bool { + // MinIO's SRPeerJoin handler force-decrypts the request body, so the + // peer/join payload must always travel encrypted. + wire_path.split_once('?').map(|(path, _)| path).unwrap_or(wire_path) == MINIO_SITE_REPLICATION_PEER_JOIN_PATH +} + +pub(crate) fn site_replication_peer_payload(path: &str, secret_key: &str, payload: Vec) -> S3Result<(Vec, &'static str)> { + if site_replication_peer_payload_encrypted(path) { + // The encrypted branch fires only for the `/minio/admin/...` peer-join + // wire path, where `crate::admin::utils::encode_compatible_admin_payload` + // unconditionally takes its compat-encryption arm — inlined here so + // this module does not import the interface layer. + let encrypted = rustfs_crypto::encrypt_stream_io(secret_key.as_bytes(), &payload) + .map_err(|e| s3_error!(InternalError, "failed to encrypt MinIO admin payload: {}", e))?; + Ok((encrypted, "application/octet-stream")) + } else { + Ok((payload, "application/json")) + } +} + +pub(crate) fn site_replication_peer_url(connection: &PeerConnection, wire_path: &str) -> S3Result { + let path = wire_path.split_once('?').map_or(wire_path, |(path, _)| path); + if !path.starts_with('/') || path.starts_with("//") { + return Err(s3_error!(InvalidRequest, "invalid site replication peer path")); + } + connection + .endpoint + .join(wire_path) + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid site replication peer path: {e}"))) +} + +#[cfg(test)] +pub(crate) async fn send_peer_admin_request_raw( + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result<(StatusCode, Vec)> { + let client = site_replication_client_for(connection).await?; + send_peer_admin_request_raw_with_client(&client, connection, path, access_key, secret_key, body).await +} + +pub(crate) async fn send_peer_admin_request_raw_with_client( + client: &reqwest::Client, + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result<(StatusCode, Vec)> { + let path = site_replication_peer_wire_path(path); + let url = site_replication_peer_url(connection, &path)?; + let uri = url + .as_str() + .parse::() + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {e}")))?; + let authority = uri + .authority() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "peer endpoint missing authority".to_string()))? + .to_string(); + let payload = serde_json::to_vec(body) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize peer request failed: {e}")))?; + let (payload, content_type) = site_replication_peer_payload(&path, secret_key, payload)?; + + let signed = sign_v4( + http::Request::builder() + .method(Method::PUT) + .uri(uri) + .header(HOST, authority) + .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) + .header(CONTENT_TYPE, content_type) + .body(Body::empty()) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build peer request failed: {e}")))?, + payload.len() as i64, + access_key, + secret_key, + "", + current_region() + .map(|region| region.to_string()) + .as_deref() + .unwrap_or("us-east-1"), + ); + + let mut req = client.request(reqwest::Method::PUT, url.clone()); + for (name, value) in signed.headers() { + req = req.header(name, value); + } + + let response = req.body(payload).send().await.map_err(|e| { + let classify = if e.is_timeout() { + "timeout" + } else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") { + "dns resolution" + } else if e.to_string().to_ascii_lowercase().contains("certificate") || e.to_string().to_ascii_lowercase().contains("tls") + { + "tls handshake" + } else if e.is_connect() { + "connect" + } else { + "request" + }; + S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) + })?; + + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("read peer response failed: {e}")))?; + + Ok((status, body.to_vec())) +} + +pub(crate) async fn send_peer_admin_request( + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result> { + let client = site_replication_client_for(connection).await?; + send_peer_admin_request_with_client(&client, connection, path, access_key, secret_key, body).await +} + +pub(crate) async fn send_peer_admin_request_with_client( + client: &reqwest::Client, + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result> { + let (status, body) = send_peer_admin_request_raw_with_client(client, connection, path, access_key, secret_key, body).await?; + if status.is_success() { + return Ok(body); + } + + let detail = String::from_utf8_lossy(&body).into_owned(); + Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer request to {}{path} failed with {status}: {detail}", connection.endpoint()), + )) +} + +pub(crate) async fn send_peer_admin_request_with_secret_candidates( + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_candidates: &[String], + body: &T, +) -> S3Result> { + let client = site_replication_client_for(connection).await?; + let mut tried = HashSet::new(); + let mut errors = Vec::new(); + + for secret_key in secret_candidates.iter().filter(|secret_key| !secret_key.is_empty()) { + if !tried.insert(secret_key.as_str()) { + continue; + } + + match send_peer_admin_request_with_client(&client, connection, path, access_key, secret_key, body).await { + Ok(body) => return Ok(body), + Err(err) => { + let detail = format!("{err}"); + let may_retry_with_next_secret = peer_error_may_be_secret_mismatch(&detail); + errors.push(summarize_peer_error_detail(&detail)); + if !may_retry_with_next_secret { + break; + } + } + } + } + + Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!( + "peer request to {}{path} failed with all service-account secrets: {}", + connection.endpoint(), + errors.join("; ") + ), + )) +} + +pub(crate) fn peer_error_may_be_secret_mismatch(detail: &str) -> bool { + let detail = detail.to_ascii_lowercase(); + detail.contains("signaturedoesnotmatch") + || detail.contains("accessdenied") + || detail.contains("forbidden") + || detail.contains("401") + || detail.contains("403") +} + +pub(crate) async fn send_peer_admin_get_request( + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_key: &str, +) -> S3Result> { + let client = site_replication_client_for(connection).await?; + send_peer_admin_get_request_with_client(&client, connection, path, access_key, secret_key).await +} + +pub(crate) async fn send_peer_admin_get_request_with_client( + client: &reqwest::Client, + connection: &PeerConnection, + path: &str, + access_key: &str, + secret_key: &str, +) -> S3Result> { + let path = site_replication_peer_wire_path(path); + let url = site_replication_peer_url(connection, &path)?; + let uri = url + .as_str() + .parse::() + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {e}")))?; + let authority = uri + .authority() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "peer endpoint missing authority".to_string()))? + .to_string(); + + let signed = sign_v4( + http::Request::builder() + .method(Method::GET) + .uri(uri) + .header(HOST, authority) + .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) + .body(Body::empty()) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build peer request failed: {e}")))?, + 0, + access_key, + secret_key, + "", + current_region() + .map(|region| region.to_string()) + .as_deref() + .unwrap_or("us-east-1"), + ); + + let mut req = client.request(reqwest::Method::GET, url.clone()); + for (name, value) in signed.headers() { + req = req.header(name, value); + } + + let response = req.send().await.map_err(|e| { + let classify = if e.is_timeout() { + "timeout" + } else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") { + "dns resolution" + } else if e.to_string().to_ascii_lowercase().contains("certificate") || e.to_string().to_ascii_lowercase().contains("tls") + { + "tls handshake" + } else if e.is_connect() { + "connect" + } else { + "request" + }; + S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) + })?; + + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("read peer response failed: {e}")))?; + + if !status.is_success() { + let detail = String::from_utf8_lossy(&body).into_owned(); + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer request to {url} failed with {status}: {detail}"), + )); + } + + Ok(body.to_vec()) +} + +pub(crate) async fn runtime_site_replication_targets() -> S3Result> { + let state = load_site_replication_state().await?; + if !state.enabled() || state.service_account_access_key.is_empty() { + return Ok(None); + } + + let service_account_secret_key = match site_replicator_service_account_secret(&state.service_account_access_key).await { + Ok(secret) => secret, + Err(err) => { + let Some(secret) = legacy_site_replicator_state_secret(&state) else { + return Err(err); + }; + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "legacy_state_service_account_secret_fallback", + error = ?err, + "admin site replication state" + ); + secret + } + }; + let local_peer = current_local_runtime_peer(&state); + Ok(Some(SiteReplicationRuntime { + state, + local_peer, + service_account_secret_key, + })) +} + +pub(crate) async fn broadcast_site_replication_json(path: &str, body: &T) -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + broadcast_site_replication_json_with_runtime(&runtime, path, body).await +} + +pub(crate) async fn broadcast_site_replication_json_with_runtime( + runtime: &SiteReplicationRuntime, + path: &str, + body: &T, +) -> S3Result<()> { + let state = &runtime.state; + let local_peer = &runtime.local_peer; + + for peer in state.peers.values() { + if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { + continue; + } + + send_peer_admin_request_with_retry_event( + peer, + path, + &state.service_account_access_key, + &runtime.service_account_secret_key, + body, + ) + .await?; + } + + Ok(()) +} + +pub(crate) async fn send_peer_admin_request_with_retry_event( + peer: &PeerInfo, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result> { + let transport = PeerTransport::for_runtime_peer(peer).await?; + send_peer_admin_request_with_retry_event_transport(peer, &transport, path, access_key, secret_key, body).await +} + +pub(crate) async fn send_peer_admin_request_with_retry_event_transport( + peer: &PeerInfo, + transport: &PeerTransport, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result> { + match send_peer_admin_request_with_client(&transport.client, &transport.connection, path, access_key, secret_key, body).await + { + Ok(body) => { + dequeue_site_replication_retry_event(peer, path).await; + Ok(body) + } + Err(err) => { + enqueue_site_replication_retry_event(peer, path, &err).await; + Err(err) + } + } +} + +pub(crate) fn parse_endpoint_refresh_status(peer: &PeerInfo, body: &[u8]) -> S3Result<()> { + let status: ReplicateEditStatus = serde_json::from_slice(body).map_err(|_| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer {} does not support endpoint target refresh", peer.endpoint), + ) + })?; + if status.success { + Ok(()) + } else { + Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer {} failed endpoint target refresh: {}", peer.endpoint, status.err_detail), + )) + } +} + +pub(crate) fn peer_capability_response_supported(peer: &PeerInfo, status: StatusCode, body: &[u8]) -> S3Result { + if status.is_success() { + return Ok(parse_endpoint_refresh_status(peer, body).is_ok()); + } + if matches!(status, StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED) { + return Ok(false); + } + + Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("probe site replication capability on peer {} failed with {status}", peer.endpoint), + )) +} + +pub(crate) fn summarize_peer_error_detail(detail: &str) -> String { + let detail = detail.trim(); + let detail_chars = detail.chars().count(); + if detail_chars <= SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT { + return detail.to_string(); + } + + let suffix = "... (truncated)"; + let take_chars = SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT.saturating_sub(suffix.chars().count()); + let mut summary: String = detail.chars().take(take_chars).collect(); + summary.push_str(suffix); + summary +} + +#[cfg(test)] +mod tests { + use super::*; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_site_replication_peer_client_rebuilds_when_generation_changes() { + let previous_generation = current_outbound_tls_generation().0; + let previous_cache = { + let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; + let snapshot = cache.clone(); + *cache = None; + snapshot + }; + + set_test_outbound_tls_generation(101); + site_replication_peer_client() + .await + .expect("initial client build should succeed"); + let cache = SITE_REPLICATION_PEER_CLIENT.lock().await; + let cached = cache.as_ref().expect("cache should be populated"); + assert_eq!(cached.generation, 101); + assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_))); + drop(cache); + + set_test_outbound_tls_generation(102); + site_replication_peer_client() + .await + .expect("new generation should rebuild client"); + let cache = SITE_REPLICATION_PEER_CLIENT.lock().await; + let cached = cache.as_ref().expect("cache should be populated"); + assert_eq!(cached.generation, 102); + assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_))); + + drop(cache); + set_test_outbound_tls_generation(previous_generation); + let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; + *cache = previous_cache; + } +} diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index cbd2ab165..80ad7fd7d 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -216,6 +216,54 @@ pub(crate) mod server { } } +/// Storage surface of the site-replication service module +/// (`crate::site_replication`, backlog#1840): bucket metadata, bucket +/// targets, replication-config primitives, and the config-object lock +/// helpers its state transaction runs on. +pub(crate) mod site_replication { + pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions}; + pub(crate) use crate::storage::storage_api::ecstore_bucket::bucket_target_sys::BucketTargetSys; + pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::{ + BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, + }; + + pub(crate) use crate::storage::storage_api::ecstore_bucket::replication::{ + OperatorRuleContract, assign_site_replication_rule_priorities, is_site_replication_role, + replication_target_arn_deployment_id, site_replication_rule_deployment_id, + }; + pub(crate) use crate::storage::storage_api::ecstore_bucket::target::{ + ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, + }; + pub(crate) use crate::storage::storage_api::ecstore_bucket::utils::{deserialize, serialize}; + pub(crate) use crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi; + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_config::com::save_config; + + pub(crate) use crate::storage::storage_api::{ + ECStore, EndpointServerPools, StorageError, delete_config_no_lock, lock_bucket_targets_metadata, read_config, + read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, + }; + + pub(crate) mod metadata_sys { + pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{ + capture_bucket_metadata_incarnation, get, get_replication_config, get_versioning_config, list_bucket_targets, + update_if_incarnation, + }; + } + + /// S3 wire types for the service module, funneled here so the module + /// itself stays off the direct s3s surface (s3s footprint ratchet). + pub(crate) mod s3 { + pub(crate) use s3s::dto::{ + BucketLifecycleConfiguration, BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, + DeleteReplication, DeleteReplicationStatus, Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, + LifecycleRule, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule, + ReplicationRuleStatus, SourceSelectionCriteria, VersioningConfiguration, + }; + pub(crate) use s3s::{Body, S3Error, S3ErrorCode, S3Response, S3Result, s3_error}; + } +} + pub(crate) mod startup { pub(crate) mod heal_control { #[cfg(test)] diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index c41c9c864..26ab34a15 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -29,6 +29,14 @@ checked_files=( "rustfs/src/admin/handlers/kms_keys.rs" "rustfs/src/admin/handlers/kms_key_lifecycle.rs" "rustfs/src/admin/handlers/site_replication.rs" + "rustfs/src/site_replication/mod.rs" + "rustfs/src/site_replication/identity.rs" + "rustfs/src/site_replication/state_lock.rs" + "rustfs/src/site_replication/state.rs" + "rustfs/src/site_replication/transport.rs" + "rustfs/src/site_replication/retry.rs" + "rustfs/src/site_replication/repair.rs" + "rustfs/src/site_replication/hooks.rs" "rustfs/src/admin/handlers/group.rs" "rustfs/src/admin/handlers/quota.rs" "rustfs/src/admin/handlers/rebalance.rs" diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index b106498e5..6132536ae 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -38,8 +38,14 @@ cd "$(dirname "$0")/.." # files, zero new s3s code — the same handler-layer surface redistributed). # The file counter is split-sensitive; the s3_error! line counter confirms # no growth (unchanged at 1620). +# 1620 → 1616 on 2026-08-27: backlog#1840 moved the site-replication service +# subsystem to rustfs/src/site_replication/ (s3s access funneled through the +# root storage facade's s3 shim, keeping the file count at 215). The move +# inlined one s3_error! call in transport.rs (+1); measured 1615 on the +# pre-move main (after #6694) and 1616 after, so the slack 1620 baseline is +# retightened to the measured 1616. S3S_IMPORT_FILES_BASELINE=215 -S3_ERROR_LINES_BASELINE=1620 +S3_ERROR_LINES_BASELINE=1616 # ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not # know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming* # client was extracted to crates/s3-client, where s3s usage is legitimate; diff --git a/scripts/layer-dependency-baseline.txt b/scripts/layer-dependency-baseline.txt index c705152a4..6483ff12c 100644 --- a/scripts/layer-dependency-baseline.txt +++ b/scripts/layer-dependency-baseline.txt @@ -18,9 +18,6 @@ cycle|app<->infra cycle|app<->interface cycle|infra<->interface dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::collect_dependency_readiness_report -dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook -dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook -dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::snapshot_dependency_readiness_report dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::server::cors